{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have finished initializing the Q-table\n",
      "Epoche=0\n",
      "Action_times=52\n",
      "Episolon=0.999485\n",
      "Epoche=1\n",
      "Action_times=80\n",
      "Episolon=0.999208\n",
      "Epoche=2\n",
      "Action_times=94\n",
      "Episolon=0.999070\n",
      "Epoche=3\n",
      "Action_times=124\n",
      "Episolon=0.998773\n",
      "Epoche=4\n",
      "Action_times=409\n",
      "Episolon=0.995959\n",
      "Epoche=5\n",
      "Action_times=519\n",
      "Episolon=0.994875\n",
      "Epoche=6\n",
      "Action_times=581\n",
      "Episolon=0.994265\n",
      "Epoche=7\n",
      "Action_times=708\n",
      "Episolon=0.993016\n",
      "Epoche=8\n",
      "Action_times=812\n",
      "Episolon=0.991994\n",
      "Epoche=9\n",
      "Action_times=934\n",
      "Episolon=0.990796\n",
      "Epoche=10\n",
      "Action_times=1197\n",
      "Episolon=0.988220\n",
      "Epoche=11\n",
      "Action_times=1256\n",
      "Episolon=0.987643\n",
      "Epoche=12\n",
      "Action_times=1312\n",
      "Episolon=0.987096\n",
      "Epoche=13\n",
      "Action_times=1394\n",
      "Episolon=0.986295\n",
      "Epoche=14\n",
      "Action_times=1434\n",
      "Episolon=0.985905\n",
      "Epoche=15\n",
      "Action_times=1454\n",
      "Episolon=0.985710\n",
      "Epoche=16\n",
      "Action_times=1598\n",
      "Episolon=0.984306\n",
      "Epoche=17\n",
      "Action_times=1649\n",
      "Episolon=0.983809\n",
      "Epoche=18\n",
      "Action_times=1677\n",
      "Episolon=0.983536\n",
      "Epoche=19\n",
      "Action_times=1710\n",
      "Episolon=0.983215\n",
      "Epoche=20\n",
      "Action_times=1837\n",
      "Episolon=0.981980\n",
      "Epoche=21\n",
      "Action_times=2015\n",
      "Episolon=0.980251\n",
      "Epoche=22\n",
      "Action_times=2099\n",
      "Episolon=0.979436\n",
      "Epoche=23\n",
      "Action_times=2270\n",
      "Episolon=0.977780\n",
      "Epoche=24\n",
      "Action_times=2282\n",
      "Episolon=0.977664\n",
      "Epoche=25\n",
      "Action_times=2366\n",
      "Episolon=0.976852\n",
      "Epoche=26\n",
      "Action_times=2648\n",
      "Episolon=0.974129\n",
      "Epoche=27\n",
      "Action_times=2882\n",
      "Episolon=0.971875\n",
      "Epoche=28\n",
      "Action_times=3137\n",
      "Episolon=0.969426\n",
      "Epoche=29\n",
      "Action_times=3207\n",
      "Episolon=0.968754\n",
      "Epoche=30\n",
      "Action_times=3282\n",
      "Episolon=0.968036\n",
      "Epoche=31\n",
      "Action_times=3301\n",
      "Episolon=0.967854\n",
      "Epoche=32\n",
      "Action_times=3341\n",
      "Episolon=0.967471\n",
      "Epoche=33\n",
      "Action_times=3399\n",
      "Episolon=0.966915\n",
      "Epoche=34\n",
      "Action_times=3466\n",
      "Episolon=0.966274\n",
      "Epoche=35\n",
      "Action_times=3555\n",
      "Episolon=0.965424\n",
      "Epoche=36\n",
      "Action_times=3582\n",
      "Episolon=0.965166\n",
      "Epoche=37\n",
      "Action_times=3638\n",
      "Episolon=0.964631\n",
      "Epoche=38\n",
      "Action_times=3713\n",
      "Episolon=0.963915\n",
      "Epoche=39\n",
      "Action_times=3729\n",
      "Episolon=0.963763\n",
      "Epoche=40\n",
      "Action_times=3777\n",
      "Episolon=0.963305\n",
      "Epoche=41\n",
      "Action_times=3782\n",
      "Episolon=0.963257\n",
      "Epoche=42\n",
      "Action_times=3913\n",
      "Episolon=0.962009\n",
      "Epoche=43\n",
      "Action_times=3998\n",
      "Episolon=0.961201\n",
      "Epoche=44\n",
      "Action_times=4006\n",
      "Episolon=0.961124\n",
      "Epoche=45\n",
      "Action_times=4038\n",
      "Episolon=0.960820\n",
      "Epoche=46\n",
      "Action_times=4048\n",
      "Episolon=0.960725\n",
      "Epoche=47\n",
      "Action_times=4182\n",
      "Episolon=0.959452\n",
      "Epoche=48\n",
      "Action_times=4310\n",
      "Episolon=0.958237\n",
      "Epoche=49\n",
      "Action_times=4364\n",
      "Episolon=0.957726\n",
      "Epoche=50\n",
      "Action_times=4376\n",
      "Episolon=0.957612\n",
      "Epoche=51\n",
      "Action_times=4418\n",
      "Episolon=0.957214\n",
      "Epoche=52\n",
      "Action_times=4463\n",
      "Episolon=0.956788\n",
      "Epoche=53\n",
      "Action_times=4465\n",
      "Episolon=0.956769\n",
      "Epoche=54\n",
      "Action_times=4567\n",
      "Episolon=0.955804\n",
      "Epoche=55\n",
      "Action_times=4640\n",
      "Episolon=0.955113\n",
      "Epoche=56\n",
      "Action_times=4703\n",
      "Episolon=0.954518\n",
      "Epoche=57\n",
      "Action_times=4779\n",
      "Episolon=0.953801\n",
      "Epoche=58\n",
      "Action_times=4868\n",
      "Episolon=0.952961\n",
      "Epoche=59\n",
      "Action_times=4869\n",
      "Episolon=0.952952\n",
      "Epoche=60\n",
      "Action_times=5270\n",
      "Episolon=0.949178\n",
      "Epoche=61\n",
      "Action_times=5297\n",
      "Episolon=0.948924\n",
      "Epoche=62\n",
      "Action_times=5382\n",
      "Episolon=0.948127\n",
      "Epoche=63\n",
      "Action_times=5416\n",
      "Episolon=0.947808\n",
      "Epoche=64\n",
      "Action_times=5523\n",
      "Episolon=0.946805\n",
      "Epoche=65\n",
      "Action_times=5682\n",
      "Episolon=0.945316\n",
      "Epoche=66\n",
      "Action_times=5801\n",
      "Episolon=0.944204\n",
      "Epoche=67\n",
      "Action_times=5857\n",
      "Episolon=0.943681\n",
      "Epoche=68\n",
      "Action_times=6056\n",
      "Episolon=0.941825\n",
      "Epoche=69\n",
      "Action_times=6113\n",
      "Episolon=0.941294\n",
      "Epoche=70\n",
      "Action_times=6147\n",
      "Episolon=0.940977\n",
      "Epoche=71\n",
      "Action_times=6178\n",
      "Episolon=0.940689\n",
      "Epoche=72\n",
      "Action_times=6413\n",
      "Episolon=0.938504\n",
      "Epoche=73\n",
      "Action_times=6545\n",
      "Episolon=0.937279\n",
      "Epoche=74\n",
      "Action_times=6573\n",
      "Episolon=0.937020\n",
      "Epoche=75\n",
      "Action_times=6626\n",
      "Episolon=0.936529\n",
      "Epoche=76\n",
      "Action_times=6787\n",
      "Episolon=0.935038\n",
      "Epoche=77\n",
      "Action_times=6844\n",
      "Episolon=0.934511\n",
      "Epoche=78\n",
      "Action_times=6900\n",
      "Episolon=0.933993\n",
      "Epoche=79\n",
      "Action_times=6954\n",
      "Episolon=0.933495\n",
      "Epoche=80\n",
      "Action_times=7007\n",
      "Episolon=0.933005\n",
      "Epoche=81\n",
      "Action_times=7033\n",
      "Episolon=0.932765\n",
      "Epoche=82\n",
      "Action_times=7142\n",
      "Episolon=0.931760\n",
      "Epoche=83\n",
      "Action_times=7224\n",
      "Episolon=0.931005\n",
      "Epoche=84\n",
      "Action_times=7284\n",
      "Episolon=0.930452\n",
      "Epoche=85\n",
      "Action_times=7373\n",
      "Episolon=0.929633\n",
      "Epoche=86\n",
      "Action_times=7430\n",
      "Episolon=0.929109\n",
      "Epoche=87\n",
      "Action_times=7507\n",
      "Episolon=0.928402\n",
      "Epoche=88\n",
      "Action_times=7516\n",
      "Episolon=0.928319\n",
      "Epoche=89\n",
      "Action_times=7639\n",
      "Episolon=0.927190\n",
      "Epoche=90\n",
      "Action_times=7700\n",
      "Episolon=0.926631\n",
      "Epoche=91\n",
      "Action_times=7753\n",
      "Episolon=0.926145\n",
      "Epoche=92\n",
      "Action_times=7822\n",
      "Episolon=0.925513\n",
      "Epoche=93\n",
      "Action_times=7849\n",
      "Episolon=0.925266\n",
      "Epoche=94\n",
      "Action_times=7927\n",
      "Episolon=0.924553\n",
      "Epoche=95\n",
      "Action_times=8118\n",
      "Episolon=0.922807\n",
      "Epoche=96\n",
      "Action_times=8165\n",
      "Episolon=0.922379\n",
      "Epoche=97\n",
      "Action_times=8235\n",
      "Episolon=0.921740\n",
      "Epoche=98\n",
      "Action_times=8251\n",
      "Episolon=0.921594\n",
      "Epoche=99\n",
      "Action_times=8342\n",
      "Episolon=0.920765\n",
      "Epoche=100\n",
      "Action_times=8410\n",
      "Episolon=0.920146\n",
      "Epoche=101\n",
      "Action_times=8475\n",
      "Episolon=0.919555\n",
      "Epoche=102\n",
      "Action_times=8548\n",
      "Episolon=0.918891\n",
      "Epoche=103\n",
      "Action_times=8577\n",
      "Episolon=0.918627\n",
      "Epoche=104\n",
      "Action_times=8678\n",
      "Episolon=0.917710\n",
      "Epoche=105\n",
      "Action_times=8727\n",
      "Episolon=0.917265\n",
      "Epoche=106\n",
      "Action_times=8748\n",
      "Episolon=0.917075\n",
      "Epoche=107\n",
      "Action_times=8765\n",
      "Episolon=0.916921\n",
      "Epoche=108\n",
      "Action_times=8859\n",
      "Episolon=0.916069\n",
      "Epoche=109\n",
      "Action_times=8953\n",
      "Episolon=0.915217\n",
      "Epoche=110\n",
      "Action_times=9178\n",
      "Episolon=0.913183\n",
      "Epoche=111\n",
      "Action_times=9192\n",
      "Episolon=0.913056\n",
      "Epoche=112\n",
      "Action_times=9321\n",
      "Episolon=0.911892\n",
      "Epoche=113\n",
      "Action_times=9348\n",
      "Episolon=0.911649\n",
      "Epoche=114\n",
      "Action_times=9435\n",
      "Episolon=0.910865\n",
      "Epoche=115\n",
      "Action_times=9517\n",
      "Episolon=0.910126\n",
      "Epoche=116\n",
      "Action_times=9631\n",
      "Episolon=0.909101\n",
      "Epoche=117\n",
      "Action_times=9711\n",
      "Episolon=0.908382\n",
      "Epoche=118\n",
      "Action_times=9792\n",
      "Episolon=0.907654\n",
      "Epoche=119\n",
      "Action_times=9849\n",
      "Episolon=0.907143\n",
      "Epoche=120\n",
      "Action_times=9914\n",
      "Episolon=0.906560\n",
      "Epoche=121\n",
      "Action_times=9915\n",
      "Episolon=0.906551\n",
      "Epoche=122\n",
      "Action_times=10005\n",
      "Episolon=0.905744\n",
      "Epoche=123\n",
      "Action_times=10063\n",
      "Episolon=0.905225\n",
      "Epoche=124\n",
      "Action_times=10133\n",
      "Episolon=0.904598\n",
      "Epoche=125\n",
      "Action_times=10142\n",
      "Episolon=0.904518\n",
      "Epoche=126\n",
      "Action_times=10221\n",
      "Episolon=0.903812\n",
      "Epoche=127\n",
      "Action_times=10313\n",
      "Episolon=0.902990\n",
      "Epoche=128\n",
      "Action_times=10383\n",
      "Episolon=0.902365\n",
      "Epoche=129\n",
      "Action_times=10407\n",
      "Episolon=0.902151\n",
      "Epoche=130\n",
      "Action_times=10454\n",
      "Episolon=0.901731\n",
      "Epoche=131\n",
      "Action_times=10495\n",
      "Episolon=0.901366\n",
      "Epoche=132\n",
      "Action_times=10525\n",
      "Episolon=0.901098\n",
      "Epoche=133\n",
      "Action_times=10689\n",
      "Episolon=0.899638\n",
      "Epoche=134\n",
      "Action_times=10765\n",
      "Episolon=0.898962\n",
      "Epoche=135\n",
      "Action_times=10809\n",
      "Episolon=0.898571\n",
      "Epoche=136\n",
      "Action_times=10958\n",
      "Episolon=0.897248\n",
      "Epoche=137\n",
      "Action_times=11088\n",
      "Episolon=0.896096\n",
      "Epoche=138\n",
      "Action_times=11231\n",
      "Episolon=0.894829\n",
      "Epoche=139\n",
      "Action_times=11292\n",
      "Episolon=0.894290\n",
      "Epoche=140\n",
      "Action_times=11359\n",
      "Episolon=0.893698\n",
      "Epoche=141\n",
      "Action_times=11376\n",
      "Episolon=0.893547\n",
      "Epoche=142\n",
      "Action_times=11459\n",
      "Episolon=0.892814\n",
      "Epoche=143\n",
      "Action_times=11488\n",
      "Episolon=0.892558\n",
      "Epoche=144\n",
      "Action_times=11567\n",
      "Episolon=0.891861\n",
      "Epoche=145\n",
      "Action_times=11579\n",
      "Episolon=0.891756\n",
      "Epoche=146\n",
      "Action_times=11664\n",
      "Episolon=0.891006\n",
      "Epoche=147\n",
      "Action_times=11720\n",
      "Episolon=0.890513\n",
      "Epoche=148\n",
      "Action_times=11799\n",
      "Episolon=0.889818\n",
      "Epoche=149\n",
      "Action_times=11869\n",
      "Episolon=0.889202\n",
      "Epoche=150\n",
      "Action_times=11952\n",
      "Episolon=0.888473\n",
      "Epoche=151\n",
      "Action_times=12017\n",
      "Episolon=0.887902\n",
      "Epoche=152\n",
      "Action_times=12178\n",
      "Episolon=0.886490\n",
      "Epoche=153\n",
      "Action_times=12217\n",
      "Episolon=0.886148\n",
      "Epoche=154\n",
      "Action_times=12345\n",
      "Episolon=0.885027\n",
      "Epoche=155\n",
      "Action_times=12377\n",
      "Episolon=0.884747\n",
      "Epoche=156\n",
      "Action_times=12451\n",
      "Episolon=0.884100\n",
      "Epoche=157\n",
      "Action_times=12593\n",
      "Episolon=0.882860\n",
      "Epoche=158\n",
      "Action_times=12606\n",
      "Episolon=0.882746\n",
      "Epoche=159\n",
      "Action_times=12638\n",
      "Episolon=0.882467\n",
      "Epoche=160\n",
      "Action_times=12806\n",
      "Episolon=0.881003\n",
      "Epoche=161\n",
      "Action_times=12857\n",
      "Episolon=0.880558\n",
      "Epoche=162\n",
      "Action_times=12909\n",
      "Episolon=0.880106\n",
      "Epoche=163\n",
      "Action_times=13008\n",
      "Episolon=0.879245\n",
      "Epoche=164\n",
      "Action_times=13037\n",
      "Episolon=0.878993\n",
      "Epoche=165\n",
      "Action_times=13080\n",
      "Episolon=0.878619\n",
      "Epoche=166\n",
      "Action_times=13184\n",
      "Episolon=0.877716\n",
      "Epoche=167\n",
      "Action_times=13261\n",
      "Episolon=0.877049\n",
      "Epoche=168\n",
      "Action_times=13286\n",
      "Episolon=0.876832\n",
      "Epoche=169\n",
      "Action_times=13293\n",
      "Episolon=0.876771\n",
      "Epoche=170\n",
      "Action_times=13330\n",
      "Episolon=0.876450\n",
      "Epoche=171\n",
      "Action_times=13391\n",
      "Episolon=0.875922\n",
      "Epoche=172\n",
      "Action_times=13407\n",
      "Episolon=0.875784\n",
      "Epoche=173\n",
      "Action_times=13465\n",
      "Episolon=0.875282\n",
      "Epoche=174\n",
      "Action_times=13527\n",
      "Episolon=0.874745\n",
      "Epoche=175\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action_times=13579\n",
      "Episolon=0.874296\n",
      "Epoche=176\n",
      "Action_times=13606\n",
      "Episolon=0.874062\n",
      "Epoche=177\n",
      "Action_times=13643\n",
      "Episolon=0.873743\n",
      "Epoche=178\n",
      "Action_times=13677\n",
      "Episolon=0.873449\n",
      "Epoche=179\n",
      "Action_times=13736\n",
      "Episolon=0.872940\n",
      "Epoche=180\n",
      "Action_times=13801\n",
      "Episolon=0.872379\n",
      "Epoche=181\n",
      "Action_times=13855\n",
      "Episolon=0.871914\n",
      "Epoche=182\n",
      "Action_times=13890\n",
      "Episolon=0.871612\n",
      "Epoche=183\n",
      "Action_times=13912\n",
      "Episolon=0.871422\n",
      "Epoche=184\n",
      "Action_times=13955\n",
      "Episolon=0.871052\n",
      "Epoche=185\n",
      "Action_times=14006\n",
      "Episolon=0.870613\n",
      "Epoche=186\n",
      "Action_times=14081\n",
      "Episolon=0.869968\n",
      "Epoche=187\n",
      "Action_times=14159\n",
      "Episolon=0.869297\n",
      "Epoche=188\n",
      "Action_times=14213\n",
      "Episolon=0.868833\n",
      "Epoche=189\n",
      "Action_times=14276\n",
      "Episolon=0.868292\n",
      "Epoche=190\n",
      "Action_times=14342\n",
      "Episolon=0.867726\n",
      "Epoche=191\n",
      "Action_times=14394\n",
      "Episolon=0.867280\n",
      "Epoche=192\n",
      "Action_times=14413\n",
      "Episolon=0.867117\n",
      "Epoche=193\n",
      "Action_times=14578\n",
      "Episolon=0.865704\n",
      "Epoche=194\n",
      "Action_times=14637\n",
      "Episolon=0.865200\n",
      "Epoche=195\n",
      "Action_times=14670\n",
      "Episolon=0.864917\n",
      "Epoche=196\n",
      "Action_times=14680\n",
      "Episolon=0.864832\n",
      "Epoche=197\n",
      "Action_times=14748\n",
      "Episolon=0.864251\n",
      "Epoche=198\n",
      "Action_times=14815\n",
      "Episolon=0.863679\n",
      "Epoche=199\n",
      "Action_times=14816\n",
      "Episolon=0.863670\n",
      "Epoche=200\n",
      "Action_times=14868\n",
      "Episolon=0.863226\n",
      "Epoche=201\n",
      "Action_times=14876\n",
      "Episolon=0.863158\n",
      "Epoche=202\n",
      "Action_times=14999\n",
      "Episolon=0.862109\n",
      "Epoche=203\n",
      "Action_times=15096\n",
      "Episolon=0.861283\n",
      "Epoche=204\n",
      "Action_times=15196\n",
      "Episolon=0.860432\n",
      "Epoche=205\n",
      "Action_times=15320\n",
      "Episolon=0.859379\n",
      "Epoche=206\n",
      "Action_times=15421\n",
      "Episolon=0.858521\n",
      "Epoche=207\n",
      "Action_times=15473\n",
      "Episolon=0.858080\n",
      "Epoche=208\n",
      "Action_times=15480\n",
      "Episolon=0.858021\n",
      "Epoche=209\n",
      "Action_times=15499\n",
      "Episolon=0.857860\n",
      "Epoche=210\n",
      "Action_times=15584\n",
      "Episolon=0.857139\n",
      "Epoche=211\n",
      "Action_times=15638\n",
      "Episolon=0.856682\n",
      "Epoche=212\n",
      "Action_times=15767\n",
      "Episolon=0.855590\n",
      "Epoche=213\n",
      "Action_times=15831\n",
      "Episolon=0.855049\n",
      "Epoche=214\n",
      "Action_times=15879\n",
      "Episolon=0.854644\n",
      "Epoche=215\n",
      "Action_times=15994\n",
      "Episolon=0.853673\n",
      "Epoche=216\n",
      "Action_times=16044\n",
      "Episolon=0.853251\n",
      "Epoche=217\n",
      "Action_times=16130\n",
      "Episolon=0.852526\n",
      "Epoche=218\n",
      "Action_times=16160\n",
      "Episolon=0.852274\n",
      "Epoche=219\n",
      "Action_times=16217\n",
      "Episolon=0.851794\n",
      "Epoche=220\n",
      "Action_times=16323\n",
      "Episolon=0.850902\n",
      "Epoche=221\n",
      "Action_times=16329\n",
      "Episolon=0.850851\n",
      "Epoche=222\n",
      "Action_times=16355\n",
      "Episolon=0.850633\n",
      "Epoche=223\n",
      "Action_times=16433\n",
      "Episolon=0.849977\n",
      "Epoche=224\n",
      "Action_times=16491\n",
      "Episolon=0.849490\n",
      "Epoche=225\n",
      "Action_times=16532\n",
      "Episolon=0.849146\n",
      "Epoche=226\n",
      "Action_times=16630\n",
      "Episolon=0.848324\n",
      "Epoche=227\n",
      "Action_times=16696\n",
      "Episolon=0.847771\n",
      "Epoche=228\n",
      "Action_times=16765\n",
      "Episolon=0.847193\n",
      "Epoche=229\n",
      "Action_times=16841\n",
      "Episolon=0.846557\n",
      "Epoche=230\n",
      "Action_times=16853\n",
      "Episolon=0.846457\n",
      "Epoche=231\n",
      "Action_times=16929\n",
      "Episolon=0.845821\n",
      "Epoche=232\n",
      "Action_times=16938\n",
      "Episolon=0.845746\n",
      "Epoche=233\n",
      "Action_times=16939\n",
      "Episolon=0.845738\n",
      "Epoche=234\n",
      "Action_times=16971\n",
      "Episolon=0.845470\n",
      "Epoche=235\n",
      "Action_times=16990\n",
      "Episolon=0.845312\n",
      "Epoche=236\n",
      "Action_times=17035\n",
      "Episolon=0.844936\n",
      "Epoche=237\n",
      "Action_times=17144\n",
      "Episolon=0.844026\n",
      "Epoche=238\n",
      "Action_times=17157\n",
      "Episolon=0.843918\n",
      "Epoche=239\n",
      "Action_times=17179\n",
      "Episolon=0.843734\n",
      "Epoche=240\n",
      "Action_times=17225\n",
      "Episolon=0.843351\n",
      "Epoche=241\n",
      "Action_times=17284\n",
      "Episolon=0.842859\n",
      "Epoche=242\n",
      "Action_times=17388\n",
      "Episolon=0.841994\n",
      "Epoche=243\n",
      "Action_times=17494\n",
      "Episolon=0.841112\n",
      "Epoche=244\n",
      "Action_times=17507\n",
      "Episolon=0.841004\n",
      "Epoche=245\n",
      "Action_times=17576\n",
      "Episolon=0.840431\n",
      "Epoche=246\n",
      "Action_times=17602\n",
      "Episolon=0.840215\n",
      "Epoche=247\n",
      "Action_times=17670\n",
      "Episolon=0.839651\n",
      "Epoche=248\n",
      "Action_times=17687\n",
      "Episolon=0.839510\n",
      "Epoche=249\n",
      "Action_times=17739\n",
      "Episolon=0.839079\n",
      "Epoche=250\n",
      "Action_times=17768\n",
      "Episolon=0.838838\n",
      "Epoche=251\n",
      "Action_times=17792\n",
      "Episolon=0.838639\n",
      "Epoche=252\n",
      "Action_times=17814\n",
      "Episolon=0.838457\n",
      "Epoche=253\n",
      "Action_times=17841\n",
      "Episolon=0.838233\n",
      "Epoche=254\n",
      "Action_times=17948\n",
      "Episolon=0.837348\n",
      "Epoche=255\n",
      "Action_times=18009\n",
      "Episolon=0.836843\n",
      "Epoche=256\n",
      "Action_times=18084\n",
      "Episolon=0.836223\n",
      "Epoche=257\n",
      "Action_times=18128\n",
      "Episolon=0.835860\n",
      "Epoche=258\n",
      "Action_times=18146\n",
      "Episolon=0.835711\n",
      "Epoche=259\n",
      "Action_times=18164\n",
      "Episolon=0.835562\n",
      "Epoche=260\n",
      "Action_times=18192\n",
      "Episolon=0.835331\n",
      "Epoche=261\n",
      "Action_times=18212\n",
      "Episolon=0.835166\n",
      "Epoche=262\n",
      "Action_times=18261\n",
      "Episolon=0.834762\n",
      "Epoche=263\n",
      "Action_times=18292\n",
      "Episolon=0.834506\n",
      "Epoche=264\n",
      "Action_times=18324\n",
      "Episolon=0.834243\n",
      "Epoche=265\n",
      "Action_times=18352\n",
      "Episolon=0.834012\n",
      "Epoche=266\n",
      "Action_times=18396\n",
      "Episolon=0.833649\n",
      "Epoche=267\n",
      "Action_times=18442\n",
      "Episolon=0.833271\n",
      "Epoche=268\n",
      "Action_times=18521\n",
      "Episolon=0.832620\n",
      "Epoche=269\n",
      "Action_times=18540\n",
      "Episolon=0.832464\n",
      "Epoche=270\n",
      "Action_times=18680\n",
      "Episolon=0.831314\n",
      "Epoche=271\n",
      "Action_times=18749\n",
      "Episolon=0.830747\n",
      "Epoche=272\n",
      "Action_times=18760\n",
      "Episolon=0.830657\n",
      "Epoche=273\n",
      "Action_times=18763\n",
      "Episolon=0.830632\n",
      "Epoche=274\n",
      "Action_times=18797\n",
      "Episolon=0.830353\n",
      "Epoche=275\n",
      "Action_times=18845\n",
      "Episolon=0.829959\n",
      "Epoche=276\n",
      "Action_times=18887\n",
      "Episolon=0.829615\n",
      "Epoche=277\n",
      "Action_times=18894\n",
      "Episolon=0.829558\n",
      "Epoche=278\n",
      "Action_times=19004\n",
      "Episolon=0.828657\n",
      "Epoche=279\n",
      "Action_times=19068\n",
      "Episolon=0.828133\n",
      "Epoche=280\n",
      "Action_times=19072\n",
      "Episolon=0.828100\n",
      "Epoche=281\n",
      "Action_times=19093\n",
      "Episolon=0.827929\n",
      "Epoche=282\n",
      "Action_times=19153\n",
      "Episolon=0.827438\n",
      "Epoche=283\n",
      "Action_times=19192\n",
      "Episolon=0.827119\n",
      "Epoche=284\n",
      "Action_times=19279\n",
      "Episolon=0.826409\n",
      "Epoche=285\n",
      "Action_times=19300\n",
      "Episolon=0.826237\n",
      "Epoche=286\n",
      "Action_times=19318\n",
      "Episolon=0.826090\n",
      "Epoche=287\n",
      "Action_times=19340\n",
      "Episolon=0.825911\n",
      "Epoche=288\n",
      "Action_times=19359\n",
      "Episolon=0.825756\n",
      "Epoche=289\n",
      "Action_times=19394\n",
      "Episolon=0.825470\n",
      "Epoche=290\n",
      "Action_times=19467\n",
      "Episolon=0.824875\n",
      "Epoche=291\n",
      "Action_times=19501\n",
      "Episolon=0.824598\n",
      "Epoche=292\n",
      "Action_times=19549\n",
      "Episolon=0.824207\n",
      "Epoche=293\n",
      "Action_times=19594\n",
      "Episolon=0.823841\n",
      "Epoche=294\n",
      "Action_times=19605\n",
      "Episolon=0.823751\n",
      "Epoche=295\n",
      "Action_times=19629\n",
      "Episolon=0.823556\n",
      "Epoche=296\n",
      "Action_times=19699\n",
      "Episolon=0.822987\n",
      "Epoche=297\n",
      "Action_times=19715\n",
      "Episolon=0.822857\n",
      "Epoche=298\n",
      "Action_times=19723\n",
      "Episolon=0.822792\n",
      "Epoche=299\n",
      "Action_times=19727\n",
      "Episolon=0.822759\n",
      "Epoche=300\n",
      "Action_times=19744\n",
      "Episolon=0.822621\n",
      "Epoche=301\n",
      "Action_times=19822\n",
      "Episolon=0.821987\n",
      "Epoche=302\n",
      "Action_times=19848\n",
      "Episolon=0.821776\n",
      "Epoche=303\n",
      "Action_times=19872\n",
      "Episolon=0.821582\n",
      "Epoche=304\n",
      "Action_times=19893\n",
      "Episolon=0.821411\n",
      "Epoche=305\n",
      "Action_times=19960\n",
      "Episolon=0.820868\n",
      "Epoche=306\n",
      "Action_times=20037\n",
      "Episolon=0.820244\n",
      "Epoche=307\n",
      "Action_times=20072\n",
      "Episolon=0.819960\n",
      "Epoche=308\n",
      "Action_times=20148\n",
      "Episolon=0.819345\n",
      "Epoche=309\n",
      "Action_times=20223\n",
      "Episolon=0.818738\n",
      "Epoche=310\n",
      "Action_times=20240\n",
      "Episolon=0.818600\n",
      "Epoche=311\n",
      "Action_times=20306\n",
      "Episolon=0.818067\n",
      "Epoche=312\n",
      "Action_times=20369\n",
      "Episolon=0.817558\n",
      "Epoche=313\n",
      "Action_times=20385\n",
      "Episolon=0.817429\n",
      "Epoche=314\n",
      "Action_times=20408\n",
      "Episolon=0.817243\n",
      "Epoche=315\n",
      "Action_times=20425\n",
      "Episolon=0.817106\n",
      "Epoche=316\n",
      "Action_times=20445\n",
      "Episolon=0.816945\n",
      "Epoche=317\n",
      "Action_times=20476\n",
      "Episolon=0.816694\n",
      "Epoche=318\n",
      "Action_times=20509\n",
      "Episolon=0.816428\n",
      "Epoche=319\n",
      "Action_times=20533\n",
      "Episolon=0.816235\n",
      "Epoche=320\n",
      "Action_times=20618\n",
      "Episolon=0.815550\n",
      "Epoche=321\n",
      "Action_times=20662\n",
      "Episolon=0.815195\n",
      "Epoche=322\n",
      "Action_times=20703\n",
      "Episolon=0.814865\n",
      "Epoche=323\n",
      "Action_times=20757\n",
      "Episolon=0.814431\n",
      "Epoche=324\n",
      "Action_times=20786\n",
      "Episolon=0.814198\n",
      "Epoche=325\n",
      "Action_times=20841\n",
      "Episolon=0.813755\n",
      "Epoche=326\n",
      "Action_times=20957\n",
      "Episolon=0.812824\n",
      "Epoche=327\n",
      "Action_times=20996\n",
      "Episolon=0.812511\n",
      "Epoche=328\n",
      "Action_times=21013\n",
      "Episolon=0.812374\n",
      "Epoche=329\n",
      "Action_times=21014\n",
      "Episolon=0.812366\n",
      "Epoche=330\n",
      "Action_times=21034\n",
      "Episolon=0.812206\n",
      "Epoche=331\n",
      "Action_times=21123\n",
      "Episolon=0.811492\n",
      "Epoche=332\n",
      "Action_times=21154\n",
      "Episolon=0.811244\n",
      "Epoche=333\n",
      "Action_times=21168\n",
      "Episolon=0.811131\n",
      "Epoche=334\n",
      "Action_times=21234\n",
      "Episolon=0.810603\n",
      "Epoche=335\n",
      "Action_times=21326\n",
      "Episolon=0.809867\n",
      "Epoche=336\n",
      "Action_times=21357\n",
      "Episolon=0.809619\n",
      "Epoche=337\n",
      "Action_times=21428\n",
      "Episolon=0.809051\n",
      "Epoche=338\n",
      "Action_times=21531\n",
      "Episolon=0.808229\n",
      "Epoche=339\n",
      "Action_times=21533\n",
      "Episolon=0.808213\n",
      "Epoche=340\n",
      "Action_times=21547\n",
      "Episolon=0.808101\n",
      "Epoche=341\n",
      "Action_times=21630\n",
      "Episolon=0.807439\n",
      "Epoche=342\n",
      "Action_times=21662\n",
      "Episolon=0.807184\n",
      "Epoche=343\n",
      "Action_times=21685\n",
      "Episolon=0.807000\n",
      "Epoche=344\n",
      "Action_times=21716\n",
      "Episolon=0.806753\n",
      "Epoche=345\n",
      "Action_times=21795\n",
      "Episolon=0.806124\n",
      "Epoche=346\n",
      "Action_times=21831\n",
      "Episolon=0.805837\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoche=347\n",
      "Action_times=21851\n",
      "Episolon=0.805678\n",
      "Epoche=348\n",
      "Action_times=21864\n",
      "Episolon=0.805575\n",
      "Epoche=349\n",
      "Action_times=21893\n",
      "Episolon=0.805344\n",
      "Epoche=350\n",
      "Action_times=21949\n",
      "Episolon=0.804899\n",
      "Epoche=351\n",
      "Action_times=21965\n",
      "Episolon=0.804772\n",
      "Epoche=352\n",
      "Action_times=21989\n",
      "Episolon=0.804581\n",
      "Epoche=353\n",
      "Action_times=22022\n",
      "Episolon=0.804319\n",
      "Epoche=354\n",
      "Action_times=22062\n",
      "Episolon=0.804001\n",
      "Epoche=355\n",
      "Action_times=22132\n",
      "Episolon=0.803446\n",
      "Epoche=356\n",
      "Action_times=22158\n",
      "Episolon=0.803239\n",
      "Epoche=357\n",
      "Action_times=22170\n",
      "Episolon=0.803144\n",
      "Epoche=358\n",
      "Action_times=22199\n",
      "Episolon=0.802914\n",
      "Epoche=359\n",
      "Action_times=22255\n",
      "Episolon=0.802470\n",
      "Epoche=360\n",
      "Action_times=22314\n",
      "Episolon=0.802003\n",
      "Epoche=361\n",
      "Action_times=22403\n",
      "Episolon=0.801298\n",
      "Epoche=362\n",
      "Action_times=22466\n",
      "Episolon=0.800800\n",
      "Epoche=363\n",
      "Action_times=22503\n",
      "Episolon=0.800507\n",
      "Epoche=364\n",
      "Action_times=22513\n",
      "Episolon=0.800428\n",
      "Epoche=365\n",
      "Action_times=22540\n",
      "Episolon=0.800215\n",
      "Epoche=366\n",
      "Action_times=22569\n",
      "Episolon=0.799986\n",
      "Epoche=367\n",
      "Action_times=22580\n",
      "Episolon=0.799899\n",
      "Epoche=368\n",
      "Action_times=22635\n",
      "Episolon=0.799465\n",
      "Epoche=369\n",
      "Action_times=22677\n",
      "Episolon=0.799133\n",
      "Epoche=370\n",
      "Action_times=22730\n",
      "Episolon=0.798715\n",
      "Epoche=371\n",
      "Action_times=22760\n",
      "Episolon=0.798478\n",
      "Epoche=372\n",
      "Action_times=22792\n",
      "Episolon=0.798226\n",
      "Epoche=373\n",
      "Action_times=22854\n",
      "Episolon=0.797738\n",
      "Epoche=374\n",
      "Action_times=22919\n",
      "Episolon=0.797226\n",
      "Epoche=375\n",
      "Action_times=22933\n",
      "Episolon=0.797115\n",
      "Epoche=376\n",
      "Action_times=22982\n",
      "Episolon=0.796730\n",
      "Epoche=377\n",
      "Action_times=23001\n",
      "Episolon=0.796580\n",
      "Epoche=378\n",
      "Action_times=23063\n",
      "Episolon=0.796093\n",
      "Epoche=379\n",
      "Action_times=23106\n",
      "Episolon=0.795755\n",
      "Epoche=380\n",
      "Action_times=23133\n",
      "Episolon=0.795543\n",
      "Epoche=381\n",
      "Action_times=23181\n",
      "Episolon=0.795166\n",
      "Epoche=382\n",
      "Action_times=23189\n",
      "Episolon=0.795103\n",
      "Epoche=383\n",
      "Action_times=23214\n",
      "Episolon=0.794907\n",
      "Epoche=384\n",
      "Action_times=23221\n",
      "Episolon=0.794852\n",
      "Epoche=385\n",
      "Action_times=23269\n",
      "Episolon=0.794475\n",
      "Epoche=386\n",
      "Action_times=23294\n",
      "Episolon=0.794279\n",
      "Epoche=387\n",
      "Action_times=23385\n",
      "Episolon=0.793566\n",
      "Epoche=388\n",
      "Action_times=23400\n",
      "Episolon=0.793448\n",
      "Epoche=389\n",
      "Action_times=23423\n",
      "Episolon=0.793268\n",
      "Epoche=390\n",
      "Action_times=23459\n",
      "Episolon=0.792986\n",
      "Epoche=391\n",
      "Action_times=23503\n",
      "Episolon=0.792642\n",
      "Epoche=392\n",
      "Action_times=23524\n",
      "Episolon=0.792477\n",
      "Epoche=393\n",
      "Action_times=23555\n",
      "Episolon=0.792235\n",
      "Epoche=394\n",
      "Action_times=23604\n",
      "Episolon=0.791852\n",
      "Epoche=395\n",
      "Action_times=23656\n",
      "Episolon=0.791445\n",
      "Epoche=396\n",
      "Action_times=23705\n",
      "Episolon=0.791062\n",
      "Epoche=397\n",
      "Action_times=23738\n",
      "Episolon=0.790805\n",
      "Epoche=398\n",
      "Action_times=23744\n",
      "Episolon=0.790758\n",
      "Epoche=399\n",
      "Action_times=23798\n",
      "Episolon=0.790336\n",
      "Epoche=400\n",
      "Action_times=23884\n",
      "Episolon=0.789665\n",
      "Epoche=401\n",
      "Action_times=23968\n",
      "Episolon=0.789011\n",
      "Epoche=402\n",
      "Action_times=24025\n",
      "Episolon=0.788567\n",
      "Epoche=403\n",
      "Action_times=24067\n",
      "Episolon=0.788240\n",
      "Epoche=404\n",
      "Action_times=24111\n",
      "Episolon=0.787898\n",
      "Epoche=405\n",
      "Action_times=24139\n",
      "Episolon=0.787680\n",
      "Epoche=406\n",
      "Action_times=24207\n",
      "Episolon=0.787151\n",
      "Epoche=407\n",
      "Action_times=24227\n",
      "Episolon=0.786996\n",
      "Epoche=408\n",
      "Action_times=24277\n",
      "Episolon=0.786607\n",
      "Epoche=409\n",
      "Action_times=24327\n",
      "Episolon=0.786219\n",
      "Epoche=410\n",
      "Action_times=24353\n",
      "Episolon=0.786017\n",
      "Epoche=411\n",
      "Action_times=24362\n",
      "Episolon=0.785948\n",
      "Epoche=412\n",
      "Action_times=24369\n",
      "Episolon=0.785893\n",
      "Epoche=413\n",
      "Action_times=24440\n",
      "Episolon=0.785343\n",
      "Epoche=414\n",
      "Action_times=24529\n",
      "Episolon=0.784653\n",
      "Epoche=415\n",
      "Action_times=24542\n",
      "Episolon=0.784552\n",
      "Epoche=416\n",
      "Action_times=24547\n",
      "Episolon=0.784513\n",
      "Epoche=417\n",
      "Action_times=24650\n",
      "Episolon=0.783716\n",
      "Epoche=418\n",
      "Action_times=24655\n",
      "Episolon=0.783677\n",
      "Epoche=419\n",
      "Action_times=24767\n",
      "Episolon=0.782811\n",
      "Epoche=420\n",
      "Action_times=24865\n",
      "Episolon=0.782054\n",
      "Epoche=421\n",
      "Action_times=24897\n",
      "Episolon=0.781807\n",
      "Epoche=422\n",
      "Action_times=24929\n",
      "Episolon=0.781560\n",
      "Epoche=423\n",
      "Action_times=24965\n",
      "Episolon=0.781283\n",
      "Epoche=424\n",
      "Action_times=24975\n",
      "Episolon=0.781206\n",
      "Epoche=425\n",
      "Action_times=25000\n",
      "Episolon=0.781013\n",
      "Epoche=426\n",
      "Action_times=25044\n",
      "Episolon=0.780674\n",
      "Epoche=427\n",
      "Action_times=25117\n",
      "Episolon=0.780111\n",
      "Epoche=428\n",
      "Action_times=25118\n",
      "Episolon=0.780104\n",
      "Epoche=429\n",
      "Action_times=25157\n",
      "Episolon=0.779803\n",
      "Epoche=430\n",
      "Action_times=25169\n",
      "Episolon=0.779711\n",
      "Epoche=431\n",
      "Action_times=25224\n",
      "Episolon=0.779288\n",
      "Epoche=432\n",
      "Action_times=25239\n",
      "Episolon=0.779172\n",
      "Epoche=433\n",
      "Action_times=25263\n",
      "Episolon=0.778988\n",
      "Epoche=434\n",
      "Action_times=25321\n",
      "Episolon=0.778542\n",
      "Epoche=435\n",
      "Action_times=25343\n",
      "Episolon=0.778373\n",
      "Epoche=436\n",
      "Action_times=25422\n",
      "Episolon=0.777766\n",
      "Epoche=437\n",
      "Action_times=25434\n",
      "Episolon=0.777674\n",
      "Epoche=438\n",
      "Action_times=25441\n",
      "Episolon=0.777620\n",
      "Epoche=439\n",
      "Action_times=25448\n",
      "Episolon=0.777566\n",
      "Epoche=440\n",
      "Action_times=25492\n",
      "Episolon=0.777229\n",
      "Epoche=441\n",
      "Action_times=25498\n",
      "Episolon=0.777183\n",
      "Epoche=442\n",
      "Action_times=25515\n",
      "Episolon=0.777052\n",
      "Epoche=443\n",
      "Action_times=25529\n",
      "Episolon=0.776945\n",
      "Epoche=444\n",
      "Action_times=25560\n",
      "Episolon=0.776707\n",
      "Epoche=445\n",
      "Action_times=25636\n",
      "Episolon=0.776125\n",
      "Epoche=446\n",
      "Action_times=25652\n",
      "Episolon=0.776002\n",
      "Epoche=447\n",
      "Action_times=25697\n",
      "Episolon=0.775658\n",
      "Epoche=448\n",
      "Action_times=25726\n",
      "Episolon=0.775435\n",
      "Epoche=449\n",
      "Action_times=25756\n",
      "Episolon=0.775206\n",
      "Epoche=450\n",
      "Action_times=25824\n",
      "Episolon=0.774686\n",
      "Epoche=451\n",
      "Action_times=25825\n",
      "Episolon=0.774678\n",
      "Epoche=452\n",
      "Action_times=25876\n",
      "Episolon=0.774288\n",
      "Epoche=453\n",
      "Action_times=25925\n",
      "Episolon=0.773914\n",
      "Epoche=454\n",
      "Action_times=26040\n",
      "Episolon=0.773036\n",
      "Epoche=455\n",
      "Action_times=26069\n",
      "Episolon=0.772815\n",
      "Epoche=456\n",
      "Action_times=26101\n",
      "Episolon=0.772570\n",
      "Epoche=457\n",
      "Action_times=26103\n",
      "Episolon=0.772555\n",
      "Epoche=458\n",
      "Action_times=26155\n",
      "Episolon=0.772159\n",
      "Epoche=459\n",
      "Action_times=26189\n",
      "Episolon=0.771900\n",
      "Epoche=460\n",
      "Action_times=26231\n",
      "Episolon=0.771580\n",
      "Epoche=461\n",
      "Action_times=26280\n",
      "Episolon=0.771207\n",
      "Epoche=462\n",
      "Action_times=26315\n",
      "Episolon=0.770940\n",
      "Epoche=463\n",
      "Action_times=26322\n",
      "Episolon=0.770887\n",
      "Epoche=464\n",
      "Action_times=26364\n",
      "Episolon=0.770568\n",
      "Epoche=465\n",
      "Action_times=26396\n",
      "Episolon=0.770324\n",
      "Epoche=466\n",
      "Action_times=26450\n",
      "Episolon=0.769914\n",
      "Epoche=467\n",
      "Action_times=26502\n",
      "Episolon=0.769519\n",
      "Epoche=468\n",
      "Action_times=26517\n",
      "Episolon=0.769405\n",
      "Epoche=469\n",
      "Action_times=26541\n",
      "Episolon=0.769223\n",
      "Epoche=470\n",
      "Action_times=26622\n",
      "Episolon=0.768608\n",
      "Epoche=471\n",
      "Action_times=26683\n",
      "Episolon=0.768145\n",
      "Epoche=472\n",
      "Action_times=26699\n",
      "Episolon=0.768024\n",
      "Epoche=473\n",
      "Action_times=26730\n",
      "Episolon=0.767789\n",
      "Epoche=474\n",
      "Action_times=26766\n",
      "Episolon=0.767516\n",
      "Epoche=475\n",
      "Action_times=26820\n",
      "Episolon=0.767107\n",
      "Epoche=476\n",
      "Action_times=26871\n",
      "Episolon=0.766721\n",
      "Epoche=477\n",
      "Action_times=26907\n",
      "Episolon=0.766449\n",
      "Epoche=478\n",
      "Action_times=26945\n",
      "Episolon=0.766161\n",
      "Epoche=479\n",
      "Action_times=26986\n",
      "Episolon=0.765852\n",
      "Epoche=480\n",
      "Action_times=26993\n",
      "Episolon=0.765799\n",
      "Epoche=481\n",
      "Action_times=27032\n",
      "Episolon=0.765504\n",
      "Epoche=482\n",
      "Action_times=27083\n",
      "Episolon=0.765119\n",
      "Epoche=483\n",
      "Action_times=27142\n",
      "Episolon=0.764673\n",
      "Epoche=484\n",
      "Action_times=27191\n",
      "Episolon=0.764304\n",
      "Epoche=485\n",
      "Action_times=27253\n",
      "Episolon=0.763836\n",
      "Epoche=486\n",
      "Action_times=27296\n",
      "Episolon=0.763512\n",
      "Epoche=487\n",
      "Action_times=27315\n",
      "Episolon=0.763369\n",
      "Epoche=488\n",
      "Action_times=27337\n",
      "Episolon=0.763203\n",
      "Epoche=489\n",
      "Action_times=27382\n",
      "Episolon=0.762864\n",
      "Epoche=490\n",
      "Action_times=27428\n",
      "Episolon=0.762518\n",
      "Epoche=491\n",
      "Action_times=27487\n",
      "Episolon=0.762074\n",
      "Epoche=492\n",
      "Action_times=27506\n",
      "Episolon=0.761931\n",
      "Epoche=493\n",
      "Action_times=27528\n",
      "Episolon=0.761766\n",
      "Epoche=494\n",
      "Action_times=27535\n",
      "Episolon=0.761713\n",
      "Epoche=495\n",
      "Action_times=27587\n",
      "Episolon=0.761322\n",
      "Epoche=496\n",
      "Action_times=27644\n",
      "Episolon=0.760894\n",
      "Epoche=497\n",
      "Action_times=27738\n",
      "Episolon=0.760189\n",
      "Epoche=498\n",
      "Action_times=27782\n",
      "Episolon=0.759859\n",
      "Epoche=499\n",
      "Action_times=27802\n",
      "Episolon=0.759709\n",
      "Epoche=500\n",
      "Action_times=27843\n",
      "Episolon=0.759402\n",
      "Epoche=501\n",
      "Action_times=27884\n",
      "Episolon=0.759094\n",
      "Epoche=502\n",
      "Action_times=27939\n",
      "Episolon=0.758682\n",
      "Epoche=503\n",
      "Action_times=27982\n",
      "Episolon=0.758361\n",
      "Epoche=504\n",
      "Action_times=27999\n",
      "Episolon=0.758233\n",
      "Epoche=505\n",
      "Action_times=28013\n",
      "Episolon=0.758129\n",
      "Epoche=506\n",
      "Action_times=28057\n",
      "Episolon=0.757800\n",
      "Epoche=507\n",
      "Action_times=28069\n",
      "Episolon=0.757710\n",
      "Epoche=508\n",
      "Action_times=28083\n",
      "Episolon=0.757605\n",
      "Epoche=509\n",
      "Action_times=28102\n",
      "Episolon=0.757463\n",
      "Epoche=510\n",
      "Action_times=28133\n",
      "Episolon=0.757231\n",
      "Epoche=511\n",
      "Action_times=28140\n",
      "Episolon=0.757179\n",
      "Epoche=512\n",
      "Action_times=28150\n",
      "Episolon=0.757104\n",
      "Epoche=513\n",
      "Action_times=28188\n",
      "Episolon=0.756821\n",
      "Epoche=514\n",
      "Action_times=28234\n",
      "Episolon=0.756477\n",
      "Epoche=515\n",
      "Action_times=28267\n",
      "Episolon=0.756231\n",
      "Epoche=516\n",
      "Action_times=28309\n",
      "Episolon=0.755917\n",
      "Epoche=517\n",
      "Action_times=28355\n",
      "Episolon=0.755574\n",
      "Epoche=518\n",
      "Action_times=28362\n",
      "Episolon=0.755522\n",
      "Epoche=519\n",
      "Action_times=28409\n",
      "Episolon=0.755172\n",
      "Epoche=520\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action_times=28418\n",
      "Episolon=0.755105\n",
      "Epoche=521\n",
      "Action_times=28494\n",
      "Episolon=0.754539\n",
      "Epoche=522\n",
      "Action_times=28543\n",
      "Episolon=0.754174\n",
      "Epoche=523\n",
      "Action_times=28591\n",
      "Episolon=0.753817\n",
      "Epoche=524\n",
      "Action_times=28640\n",
      "Episolon=0.753453\n",
      "Epoche=525\n",
      "Action_times=28649\n",
      "Episolon=0.753386\n",
      "Epoche=526\n",
      "Action_times=28699\n",
      "Episolon=0.753014\n",
      "Epoche=527\n",
      "Action_times=28707\n",
      "Episolon=0.752955\n",
      "Epoche=528\n",
      "Action_times=28733\n",
      "Episolon=0.752761\n",
      "Epoche=529\n",
      "Action_times=28755\n",
      "Episolon=0.752598\n",
      "Epoche=530\n",
      "Action_times=28798\n",
      "Episolon=0.752279\n",
      "Epoche=531\n",
      "Action_times=28854\n",
      "Episolon=0.751863\n",
      "Epoche=532\n",
      "Action_times=28884\n",
      "Episolon=0.751641\n",
      "Epoche=533\n",
      "Action_times=28885\n",
      "Episolon=0.751633\n",
      "Epoche=534\n",
      "Action_times=28904\n",
      "Episolon=0.751492\n",
      "Epoche=535\n",
      "Action_times=28949\n",
      "Episolon=0.751159\n",
      "Epoche=536\n",
      "Action_times=29013\n",
      "Episolon=0.750685\n",
      "Epoche=537\n",
      "Action_times=29070\n",
      "Episolon=0.750263\n",
      "Epoche=538\n",
      "Action_times=29098\n",
      "Episolon=0.750055\n",
      "Epoche=539\n",
      "Action_times=29150\n",
      "Episolon=0.749671\n",
      "Epoche=540\n",
      "Action_times=29176\n",
      "Episolon=0.749478\n",
      "Epoche=541\n",
      "Action_times=29226\n",
      "Episolon=0.749109\n",
      "Epoche=542\n",
      "Action_times=29263\n",
      "Episolon=0.748835\n",
      "Epoche=543\n",
      "Action_times=29315\n",
      "Episolon=0.748451\n",
      "Epoche=544\n",
      "Action_times=29336\n",
      "Episolon=0.748296\n",
      "Epoche=545\n",
      "Action_times=29379\n",
      "Episolon=0.747979\n",
      "Epoche=546\n",
      "Action_times=29387\n",
      "Episolon=0.747920\n",
      "Epoche=547\n",
      "Action_times=29394\n",
      "Episolon=0.747868\n",
      "Epoche=548\n",
      "Action_times=29428\n",
      "Episolon=0.747617\n",
      "Epoche=549\n",
      "Action_times=29441\n",
      "Episolon=0.747521\n",
      "Epoche=550\n",
      "Action_times=29486\n",
      "Episolon=0.747189\n",
      "Epoche=551\n",
      "Action_times=29496\n",
      "Episolon=0.747116\n",
      "Epoche=552\n",
      "Action_times=29514\n",
      "Episolon=0.746983\n",
      "Epoche=553\n",
      "Action_times=29540\n",
      "Episolon=0.746791\n",
      "Epoche=554\n",
      "Action_times=29627\n",
      "Episolon=0.746151\n",
      "Epoche=555\n",
      "Action_times=29652\n",
      "Episolon=0.745967\n",
      "Epoche=556\n",
      "Action_times=29672\n",
      "Episolon=0.745820\n",
      "Epoche=557\n",
      "Action_times=29706\n",
      "Episolon=0.745569\n",
      "Epoche=558\n",
      "Action_times=29736\n",
      "Episolon=0.745349\n",
      "Epoche=559\n",
      "Action_times=29798\n",
      "Episolon=0.744893\n",
      "Epoche=560\n",
      "Action_times=29841\n",
      "Episolon=0.744577\n",
      "Epoche=561\n",
      "Action_times=29874\n",
      "Episolon=0.744335\n",
      "Epoche=562\n",
      "Action_times=29910\n",
      "Episolon=0.744070\n",
      "Epoche=563\n",
      "Action_times=29921\n",
      "Episolon=0.743990\n",
      "Epoche=564\n",
      "Action_times=29949\n",
      "Episolon=0.743784\n",
      "Epoche=565\n",
      "Action_times=29972\n",
      "Episolon=0.743615\n",
      "Epoche=566\n",
      "Action_times=29987\n",
      "Episolon=0.743505\n",
      "Epoche=567\n",
      "Action_times=30035\n",
      "Episolon=0.743153\n",
      "Epoche=568\n",
      "Action_times=30099\n",
      "Episolon=0.742684\n",
      "Epoche=569\n",
      "Action_times=30159\n",
      "Episolon=0.742245\n",
      "Epoche=570\n",
      "Action_times=30200\n",
      "Episolon=0.741945\n",
      "Epoche=571\n",
      "Action_times=30231\n",
      "Episolon=0.741718\n",
      "Epoche=572\n",
      "Action_times=30244\n",
      "Episolon=0.741623\n",
      "Epoche=573\n",
      "Action_times=30319\n",
      "Episolon=0.741074\n",
      "Epoche=574\n",
      "Action_times=30381\n",
      "Episolon=0.740621\n",
      "Epoche=575\n",
      "Action_times=30409\n",
      "Episolon=0.740417\n",
      "Epoche=576\n",
      "Action_times=30450\n",
      "Episolon=0.740117\n",
      "Epoche=577\n",
      "Action_times=30453\n",
      "Episolon=0.740095\n",
      "Epoche=578\n",
      "Action_times=30465\n",
      "Episolon=0.740008\n",
      "Epoche=579\n",
      "Action_times=30517\n",
      "Episolon=0.739628\n",
      "Epoche=580\n",
      "Action_times=30552\n",
      "Episolon=0.739373\n",
      "Epoche=581\n",
      "Action_times=30571\n",
      "Episolon=0.739234\n",
      "Epoche=582\n",
      "Action_times=30595\n",
      "Episolon=0.739059\n",
      "Epoche=583\n",
      "Action_times=30711\n",
      "Episolon=0.738214\n",
      "Epoche=584\n",
      "Action_times=30808\n",
      "Episolon=0.737508\n",
      "Epoche=585\n",
      "Action_times=30836\n",
      "Episolon=0.737304\n",
      "Epoche=586\n",
      "Action_times=30878\n",
      "Episolon=0.736999\n",
      "Epoche=587\n",
      "Action_times=30890\n",
      "Episolon=0.736912\n",
      "Epoche=588\n",
      "Action_times=30901\n",
      "Episolon=0.736832\n",
      "Epoche=589\n",
      "Action_times=30932\n",
      "Episolon=0.736606\n",
      "Epoche=590\n",
      "Action_times=30954\n",
      "Episolon=0.736447\n",
      "Epoche=591\n",
      "Action_times=30985\n",
      "Episolon=0.736221\n",
      "Epoche=592\n",
      "Action_times=30991\n",
      "Episolon=0.736178\n",
      "Epoche=593\n",
      "Action_times=31020\n",
      "Episolon=0.735967\n",
      "Epoche=594\n",
      "Action_times=31039\n",
      "Episolon=0.735829\n",
      "Epoche=595\n",
      "Action_times=31078\n",
      "Episolon=0.735546\n",
      "Epoche=596\n",
      "Action_times=31105\n",
      "Episolon=0.735350\n",
      "Epoche=597\n",
      "Action_times=31148\n",
      "Episolon=0.735039\n",
      "Epoche=598\n",
      "Action_times=31153\n",
      "Episolon=0.735002\n",
      "Epoche=599\n",
      "Action_times=31187\n",
      "Episolon=0.734756\n",
      "Epoche=600\n",
      "Action_times=31205\n",
      "Episolon=0.734625\n",
      "Epoche=601\n",
      "Action_times=31235\n",
      "Episolon=0.734408\n",
      "Epoche=602\n",
      "Action_times=31340\n",
      "Episolon=0.733648\n",
      "Epoche=603\n",
      "Action_times=31379\n",
      "Episolon=0.733366\n",
      "Epoche=604\n",
      "Action_times=31460\n",
      "Episolon=0.732780\n",
      "Epoche=605\n",
      "Action_times=31478\n",
      "Episolon=0.732650\n",
      "Epoche=606\n",
      "Action_times=31512\n",
      "Episolon=0.732404\n",
      "Epoche=607\n",
      "Action_times=31541\n",
      "Episolon=0.732195\n",
      "Epoche=608\n",
      "Action_times=31560\n",
      "Episolon=0.732058\n",
      "Epoche=609\n",
      "Action_times=31597\n",
      "Episolon=0.731791\n",
      "Epoche=610\n",
      "Action_times=31612\n",
      "Episolon=0.731682\n",
      "Epoche=611\n",
      "Action_times=31632\n",
      "Episolon=0.731538\n",
      "Epoche=612\n",
      "Action_times=31649\n",
      "Episolon=0.731415\n",
      "Epoche=613\n",
      "Action_times=31657\n",
      "Episolon=0.731358\n",
      "Epoche=614\n",
      "Action_times=31664\n",
      "Episolon=0.731307\n",
      "Epoche=615\n",
      "Action_times=31751\n",
      "Episolon=0.730680\n",
      "Epoche=616\n",
      "Action_times=31765\n",
      "Episolon=0.730579\n",
      "Epoche=617\n",
      "Action_times=31827\n",
      "Episolon=0.730132\n",
      "Epoche=618\n",
      "Action_times=31886\n",
      "Episolon=0.729708\n",
      "Epoche=619\n",
      "Action_times=31907\n",
      "Episolon=0.729556\n",
      "Epoche=620\n",
      "Action_times=31938\n",
      "Episolon=0.729333\n",
      "Epoche=621\n",
      "Action_times=32000\n",
      "Episolon=0.728888\n",
      "Epoche=622\n",
      "Action_times=32057\n",
      "Episolon=0.728478\n",
      "Epoche=623\n",
      "Action_times=32105\n",
      "Episolon=0.728133\n",
      "Epoche=624\n",
      "Action_times=32139\n",
      "Episolon=0.727889\n",
      "Epoche=625\n",
      "Action_times=32159\n",
      "Episolon=0.727745\n",
      "Epoche=626\n",
      "Action_times=32212\n",
      "Episolon=0.727365\n",
      "Epoche=627\n",
      "Action_times=32282\n",
      "Episolon=0.726863\n",
      "Epoche=628\n",
      "Action_times=32326\n",
      "Episolon=0.726548\n",
      "Epoche=629\n",
      "Action_times=32367\n",
      "Episolon=0.726254\n",
      "Epoche=630\n",
      "Action_times=32413\n",
      "Episolon=0.725925\n",
      "Epoche=631\n",
      "Action_times=32414\n",
      "Episolon=0.725918\n",
      "Epoche=632\n",
      "Action_times=32428\n",
      "Episolon=0.725817\n",
      "Epoche=633\n",
      "Action_times=32496\n",
      "Episolon=0.725331\n",
      "Epoche=634\n",
      "Action_times=32540\n",
      "Episolon=0.725016\n",
      "Epoche=635\n",
      "Action_times=32550\n",
      "Episolon=0.724945\n",
      "Epoche=636\n",
      "Action_times=32580\n",
      "Episolon=0.724730\n",
      "Epoche=637\n",
      "Action_times=32656\n",
      "Episolon=0.724187\n",
      "Epoche=638\n",
      "Action_times=32659\n",
      "Episolon=0.724166\n",
      "Epoche=639\n",
      "Action_times=32681\n",
      "Episolon=0.724009\n",
      "Epoche=640\n",
      "Action_times=32703\n",
      "Episolon=0.723851\n",
      "Epoche=641\n",
      "Action_times=32769\n",
      "Episolon=0.723381\n",
      "Epoche=642\n",
      "Action_times=32808\n",
      "Episolon=0.723102\n",
      "Epoche=643\n",
      "Action_times=32857\n",
      "Episolon=0.722753\n",
      "Epoche=644\n",
      "Action_times=32862\n",
      "Episolon=0.722717\n",
      "Epoche=645\n",
      "Action_times=32896\n",
      "Episolon=0.722475\n",
      "Epoche=646\n",
      "Action_times=32937\n",
      "Episolon=0.722183\n",
      "Epoche=647\n",
      "Action_times=32985\n",
      "Episolon=0.721841\n",
      "Epoche=648\n",
      "Action_times=33019\n",
      "Episolon=0.721599\n",
      "Epoche=649\n",
      "Action_times=33048\n",
      "Episolon=0.721393\n",
      "Epoche=650\n",
      "Action_times=33066\n",
      "Episolon=0.721265\n",
      "Epoche=651\n",
      "Action_times=33098\n",
      "Episolon=0.721037\n",
      "Epoche=652\n",
      "Action_times=33173\n",
      "Episolon=0.720504\n",
      "Epoche=653\n",
      "Action_times=33192\n",
      "Episolon=0.720369\n",
      "Epoche=654\n",
      "Action_times=33228\n",
      "Episolon=0.720114\n",
      "Epoche=655\n",
      "Action_times=33266\n",
      "Episolon=0.719844\n",
      "Epoche=656\n",
      "Action_times=33298\n",
      "Episolon=0.719617\n",
      "Epoche=657\n",
      "Action_times=33306\n",
      "Episolon=0.719560\n",
      "Epoche=658\n",
      "Action_times=33318\n",
      "Episolon=0.719475\n",
      "Epoche=659\n",
      "Action_times=33372\n",
      "Episolon=0.719092\n",
      "Epoche=660\n",
      "Action_times=33384\n",
      "Episolon=0.719007\n",
      "Epoche=661\n",
      "Action_times=33427\n",
      "Episolon=0.718702\n",
      "Epoche=662\n",
      "Action_times=33436\n",
      "Episolon=0.718638\n",
      "Epoche=663\n",
      "Action_times=33444\n",
      "Episolon=0.718581\n",
      "Epoche=664\n",
      "Action_times=33446\n",
      "Episolon=0.718567\n",
      "Epoche=665\n",
      "Action_times=33491\n",
      "Episolon=0.718248\n",
      "Epoche=666\n",
      "Action_times=33522\n",
      "Episolon=0.718029\n",
      "Epoche=667\n",
      "Action_times=33534\n",
      "Episolon=0.717944\n",
      "Epoche=668\n",
      "Action_times=33586\n",
      "Episolon=0.717576\n",
      "Epoche=669\n",
      "Action_times=33589\n",
      "Episolon=0.717555\n",
      "Epoche=670\n",
      "Action_times=33641\n",
      "Episolon=0.717187\n",
      "Epoche=671\n",
      "Action_times=33669\n",
      "Episolon=0.716989\n",
      "Epoche=672\n",
      "Action_times=33686\n",
      "Episolon=0.716869\n",
      "Epoche=673\n",
      "Action_times=33724\n",
      "Episolon=0.716600\n",
      "Epoche=674\n",
      "Action_times=33741\n",
      "Episolon=0.716480\n",
      "Epoche=675\n",
      "Action_times=33787\n",
      "Episolon=0.716155\n",
      "Epoche=676\n",
      "Action_times=33816\n",
      "Episolon=0.715950\n",
      "Epoche=677\n",
      "Action_times=33837\n",
      "Episolon=0.715802\n",
      "Epoche=678\n",
      "Action_times=33898\n",
      "Episolon=0.715372\n",
      "Epoche=679\n",
      "Action_times=33926\n",
      "Episolon=0.715174\n",
      "Epoche=680\n",
      "Action_times=33965\n",
      "Episolon=0.714899\n",
      "Epoche=681\n",
      "Action_times=34002\n",
      "Episolon=0.714639\n",
      "Epoche=682\n",
      "Action_times=34025\n",
      "Episolon=0.714476\n",
      "Epoche=683\n",
      "Action_times=34056\n",
      "Episolon=0.714258\n",
      "Epoche=684\n",
      "Action_times=34122\n",
      "Episolon=0.713793\n",
      "Epoche=685\n",
      "Action_times=34164\n",
      "Episolon=0.713498\n",
      "Epoche=686\n",
      "Action_times=34210\n",
      "Episolon=0.713174\n",
      "Epoche=687\n",
      "Action_times=34222\n",
      "Episolon=0.713090\n",
      "Epoche=688\n",
      "Action_times=34272\n",
      "Episolon=0.712739\n",
      "Epoche=689\n",
      "Action_times=34306\n",
      "Episolon=0.712500\n",
      "Epoche=690\n",
      "Action_times=34328\n",
      "Episolon=0.712345\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoche=691\n",
      "Action_times=34375\n",
      "Episolon=0.712015\n",
      "Epoche=692\n",
      "Action_times=34429\n",
      "Episolon=0.711636\n",
      "Epoche=693\n",
      "Action_times=34449\n",
      "Episolon=0.711496\n",
      "Epoche=694\n",
      "Action_times=34466\n",
      "Episolon=0.711377\n",
      "Epoche=695\n",
      "Action_times=34483\n",
      "Episolon=0.711257\n",
      "Epoche=696\n",
      "Action_times=34497\n",
      "Episolon=0.711159\n",
      "Epoche=697\n",
      "Action_times=34535\n",
      "Episolon=0.710893\n",
      "Epoche=698\n",
      "Action_times=34596\n",
      "Episolon=0.710465\n",
      "Epoche=699\n",
      "Action_times=34607\n",
      "Episolon=0.710388\n",
      "Epoche=700\n",
      "Action_times=34634\n",
      "Episolon=0.710199\n",
      "Epoche=701\n",
      "Action_times=34702\n",
      "Episolon=0.709723\n",
      "Epoche=702\n",
      "Action_times=34742\n",
      "Episolon=0.709443\n",
      "Epoche=703\n",
      "Action_times=34784\n",
      "Episolon=0.709150\n",
      "Epoche=704\n",
      "Action_times=34797\n",
      "Episolon=0.709059\n",
      "Epoche=705\n",
      "Action_times=34827\n",
      "Episolon=0.708849\n",
      "Epoche=706\n",
      "Action_times=34883\n",
      "Episolon=0.708458\n",
      "Epoche=707\n",
      "Action_times=34919\n",
      "Episolon=0.708207\n",
      "Epoche=708\n",
      "Action_times=34939\n",
      "Episolon=0.708067\n",
      "Epoche=709\n",
      "Action_times=34957\n",
      "Episolon=0.707941\n",
      "Epoche=710\n",
      "Action_times=34980\n",
      "Episolon=0.707781\n",
      "Epoche=711\n",
      "Action_times=34981\n",
      "Episolon=0.707774\n",
      "Epoche=712\n",
      "Action_times=35029\n",
      "Episolon=0.707439\n",
      "Epoche=713\n",
      "Action_times=35101\n",
      "Episolon=0.706937\n",
      "Epoche=714\n",
      "Action_times=35154\n",
      "Episolon=0.706568\n",
      "Epoche=715\n",
      "Action_times=35188\n",
      "Episolon=0.706331\n",
      "Epoche=716\n",
      "Action_times=35232\n",
      "Episolon=0.706025\n",
      "Epoche=717\n",
      "Action_times=35295\n",
      "Episolon=0.705586\n",
      "Epoche=718\n",
      "Action_times=35358\n",
      "Episolon=0.705148\n",
      "Epoche=719\n",
      "Action_times=35385\n",
      "Episolon=0.704960\n",
      "Epoche=720\n",
      "Action_times=35418\n",
      "Episolon=0.704731\n",
      "Epoche=721\n",
      "Action_times=35440\n",
      "Episolon=0.704578\n",
      "Epoche=722\n",
      "Action_times=35448\n",
      "Episolon=0.704523\n",
      "Epoche=723\n",
      "Action_times=35454\n",
      "Episolon=0.704481\n",
      "Epoche=724\n",
      "Action_times=35499\n",
      "Episolon=0.704169\n",
      "Epoche=725\n",
      "Action_times=35511\n",
      "Episolon=0.704085\n",
      "Epoche=726\n",
      "Action_times=35556\n",
      "Episolon=0.703773\n",
      "Epoche=727\n",
      "Action_times=35608\n",
      "Episolon=0.703412\n",
      "Epoche=728\n",
      "Action_times=35637\n",
      "Episolon=0.703211\n",
      "Epoche=729\n",
      "Action_times=35644\n",
      "Episolon=0.703163\n",
      "Epoche=730\n",
      "Action_times=35683\n",
      "Episolon=0.702893\n",
      "Epoche=731\n",
      "Action_times=35714\n",
      "Episolon=0.702678\n",
      "Epoche=732\n",
      "Action_times=35730\n",
      "Episolon=0.702567\n",
      "Epoche=733\n",
      "Action_times=35749\n",
      "Episolon=0.702435\n",
      "Epoche=734\n",
      "Action_times=35762\n",
      "Episolon=0.702345\n",
      "Epoche=735\n",
      "Action_times=35797\n",
      "Episolon=0.702103\n",
      "Epoche=736\n",
      "Action_times=35834\n",
      "Episolon=0.701847\n",
      "Epoche=737\n",
      "Action_times=35875\n",
      "Episolon=0.701563\n",
      "Epoche=738\n",
      "Action_times=35888\n",
      "Episolon=0.701474\n",
      "Epoche=739\n",
      "Action_times=35962\n",
      "Episolon=0.700962\n",
      "Epoche=740\n",
      "Action_times=36024\n",
      "Episolon=0.700534\n",
      "Epoche=741\n",
      "Action_times=36029\n",
      "Episolon=0.700499\n",
      "Epoche=742\n",
      "Action_times=36067\n",
      "Episolon=0.700237\n",
      "Epoche=743\n",
      "Action_times=36115\n",
      "Episolon=0.699906\n",
      "Epoche=744\n",
      "Action_times=36134\n",
      "Episolon=0.699775\n",
      "Epoche=745\n",
      "Action_times=36162\n",
      "Episolon=0.699582\n",
      "Epoche=746\n",
      "Action_times=36184\n",
      "Episolon=0.699430\n",
      "Epoche=747\n",
      "Action_times=36188\n",
      "Episolon=0.699402\n",
      "Epoche=748\n",
      "Action_times=36206\n",
      "Episolon=0.699278\n",
      "Epoche=749\n",
      "Action_times=36252\n",
      "Episolon=0.698961\n",
      "Epoche=750\n",
      "Action_times=36277\n",
      "Episolon=0.698789\n",
      "Epoche=751\n",
      "Action_times=36305\n",
      "Episolon=0.698596\n",
      "Epoche=752\n",
      "Action_times=36351\n",
      "Episolon=0.698279\n",
      "Epoche=753\n",
      "Action_times=36413\n",
      "Episolon=0.697853\n",
      "Epoche=754\n",
      "Action_times=36436\n",
      "Episolon=0.697695\n",
      "Epoche=755\n",
      "Action_times=36453\n",
      "Episolon=0.697578\n",
      "Epoche=756\n",
      "Action_times=36458\n",
      "Episolon=0.697543\n",
      "Epoche=757\n",
      "Action_times=36464\n",
      "Episolon=0.697502\n",
      "Epoche=758\n",
      "Action_times=36468\n",
      "Episolon=0.697475\n",
      "Epoche=759\n",
      "Action_times=36469\n",
      "Episolon=0.697468\n",
      "Epoche=760\n",
      "Action_times=36489\n",
      "Episolon=0.697330\n",
      "Epoche=761\n",
      "Action_times=36556\n",
      "Episolon=0.696870\n",
      "Epoche=762\n",
      "Action_times=36573\n",
      "Episolon=0.696753\n",
      "Epoche=763\n",
      "Action_times=36652\n",
      "Episolon=0.696211\n",
      "Epoche=764\n",
      "Action_times=36699\n",
      "Episolon=0.695888\n",
      "Epoche=765\n",
      "Action_times=36722\n",
      "Episolon=0.695731\n",
      "Epoche=766\n",
      "Action_times=36769\n",
      "Episolon=0.695408\n",
      "Epoche=767\n",
      "Action_times=36782\n",
      "Episolon=0.695319\n",
      "Epoche=768\n",
      "Action_times=36786\n",
      "Episolon=0.695292\n",
      "Epoche=769\n",
      "Action_times=36806\n",
      "Episolon=0.695155\n",
      "Epoche=770\n",
      "Action_times=36858\n",
      "Episolon=0.694799\n",
      "Epoche=771\n",
      "Action_times=36902\n",
      "Episolon=0.694497\n",
      "Epoche=772\n",
      "Action_times=36940\n",
      "Episolon=0.694237\n",
      "Epoche=773\n",
      "Action_times=36966\n",
      "Episolon=0.694060\n",
      "Epoche=774\n",
      "Action_times=36999\n",
      "Episolon=0.693834\n",
      "Epoche=775\n",
      "Action_times=37005\n",
      "Episolon=0.693793\n",
      "Epoche=776\n",
      "Action_times=37062\n",
      "Episolon=0.693403\n",
      "Epoche=777\n",
      "Action_times=37100\n",
      "Episolon=0.693144\n",
      "Epoche=778\n",
      "Action_times=37144\n",
      "Episolon=0.692843\n",
      "Epoche=779\n",
      "Action_times=37155\n",
      "Episolon=0.692768\n",
      "Epoche=780\n",
      "Action_times=37181\n",
      "Episolon=0.692590\n",
      "Epoche=781\n",
      "Action_times=37189\n",
      "Episolon=0.692536\n",
      "Epoche=782\n",
      "Action_times=37218\n",
      "Episolon=0.692338\n",
      "Epoche=783\n",
      "Action_times=37229\n",
      "Episolon=0.692263\n",
      "Epoche=784\n",
      "Action_times=37276\n",
      "Episolon=0.691942\n",
      "Epoche=785\n",
      "Action_times=37304\n",
      "Episolon=0.691751\n",
      "Epoche=786\n",
      "Action_times=37323\n",
      "Episolon=0.691622\n",
      "Epoche=787\n",
      "Action_times=37363\n",
      "Episolon=0.691349\n",
      "Epoche=788\n",
      "Action_times=37377\n",
      "Episolon=0.691254\n",
      "Epoche=789\n",
      "Action_times=37413\n",
      "Episolon=0.691009\n",
      "Epoche=790\n",
      "Action_times=37465\n",
      "Episolon=0.690655\n",
      "Epoche=791\n",
      "Action_times=37471\n",
      "Episolon=0.690614\n",
      "Epoche=792\n",
      "Action_times=37534\n",
      "Episolon=0.690185\n",
      "Epoche=793\n",
      "Action_times=37575\n",
      "Episolon=0.689906\n",
      "Epoche=794\n",
      "Action_times=37618\n",
      "Episolon=0.689614\n",
      "Epoche=795\n",
      "Action_times=37651\n",
      "Episolon=0.689390\n",
      "Epoche=796\n",
      "Action_times=37706\n",
      "Episolon=0.689016\n",
      "Epoche=797\n",
      "Action_times=37740\n",
      "Episolon=0.688785\n",
      "Epoche=798\n",
      "Action_times=37765\n",
      "Episolon=0.688616\n",
      "Epoche=799\n",
      "Action_times=37792\n",
      "Episolon=0.688432\n",
      "Epoche=800\n",
      "Action_times=37826\n",
      "Episolon=0.688202\n",
      "Epoche=801\n",
      "Action_times=37846\n",
      "Episolon=0.688066\n",
      "Epoche=802\n",
      "Action_times=37865\n",
      "Episolon=0.687937\n",
      "Epoche=803\n",
      "Action_times=37897\n",
      "Episolon=0.687720\n",
      "Epoche=804\n",
      "Action_times=37941\n",
      "Episolon=0.687422\n",
      "Epoche=805\n",
      "Action_times=37994\n",
      "Episolon=0.687063\n",
      "Epoche=806\n",
      "Action_times=38006\n",
      "Episolon=0.686982\n",
      "Epoche=807\n",
      "Action_times=38022\n",
      "Episolon=0.686874\n",
      "Epoche=808\n",
      "Action_times=38043\n",
      "Episolon=0.686732\n",
      "Epoche=809\n",
      "Action_times=38074\n",
      "Episolon=0.686522\n",
      "Epoche=810\n",
      "Action_times=38103\n",
      "Episolon=0.686326\n",
      "Epoche=811\n",
      "Action_times=38132\n",
      "Episolon=0.686130\n",
      "Epoche=812\n",
      "Action_times=38146\n",
      "Episolon=0.686035\n",
      "Epoche=813\n",
      "Action_times=38162\n",
      "Episolon=0.685927\n",
      "Epoche=814\n",
      "Action_times=38215\n",
      "Episolon=0.685569\n",
      "Epoche=815\n",
      "Action_times=38271\n",
      "Episolon=0.685191\n",
      "Epoche=816\n",
      "Action_times=38281\n",
      "Episolon=0.685123\n",
      "Epoche=817\n",
      "Action_times=38290\n",
      "Episolon=0.685062\n",
      "Epoche=818\n",
      "Action_times=38297\n",
      "Episolon=0.685015\n",
      "Epoche=819\n",
      "Action_times=38311\n",
      "Episolon=0.684921\n",
      "Epoche=820\n",
      "Action_times=38345\n",
      "Episolon=0.684691\n",
      "Epoche=821\n",
      "Action_times=38362\n",
      "Episolon=0.684576\n",
      "Epoche=822\n",
      "Action_times=38389\n",
      "Episolon=0.684394\n",
      "Epoche=823\n",
      "Action_times=38424\n",
      "Episolon=0.684158\n",
      "Epoche=824\n",
      "Action_times=38425\n",
      "Episolon=0.684152\n",
      "Epoche=825\n",
      "Action_times=38445\n",
      "Episolon=0.684017\n",
      "Epoche=826\n",
      "Action_times=38489\n",
      "Episolon=0.683720\n",
      "Epoche=827\n",
      "Action_times=38536\n",
      "Episolon=0.683404\n",
      "Epoche=828\n",
      "Action_times=38543\n",
      "Episolon=0.683357\n",
      "Epoche=829\n",
      "Action_times=38579\n",
      "Episolon=0.683114\n",
      "Epoche=830\n",
      "Action_times=38609\n",
      "Episolon=0.682912\n",
      "Epoche=831\n",
      "Action_times=38616\n",
      "Episolon=0.682865\n",
      "Epoche=832\n",
      "Action_times=38628\n",
      "Episolon=0.682784\n",
      "Epoche=833\n",
      "Action_times=38652\n",
      "Episolon=0.682623\n",
      "Epoche=834\n",
      "Action_times=38708\n",
      "Episolon=0.682246\n",
      "Epoche=835\n",
      "Action_times=38735\n",
      "Episolon=0.682065\n",
      "Epoche=836\n",
      "Action_times=38754\n",
      "Episolon=0.681937\n",
      "Epoche=837\n",
      "Action_times=38785\n",
      "Episolon=0.681729\n",
      "Epoche=838\n",
      "Action_times=38816\n",
      "Episolon=0.681521\n",
      "Epoche=839\n",
      "Action_times=38825\n",
      "Episolon=0.681460\n",
      "Epoche=840\n",
      "Action_times=38851\n",
      "Episolon=0.681286\n",
      "Epoche=841\n",
      "Action_times=38861\n",
      "Episolon=0.681219\n",
      "Epoche=842\n",
      "Action_times=38877\n",
      "Episolon=0.681111\n",
      "Epoche=843\n",
      "Action_times=38897\n",
      "Episolon=0.680977\n",
      "Epoche=844\n",
      "Action_times=38907\n",
      "Episolon=0.680910\n",
      "Epoche=845\n",
      "Action_times=38939\n",
      "Episolon=0.680695\n",
      "Epoche=846\n",
      "Action_times=38966\n",
      "Episolon=0.680514\n",
      "Epoche=847\n",
      "Action_times=39008\n",
      "Episolon=0.680233\n",
      "Epoche=848\n",
      "Action_times=39044\n",
      "Episolon=0.679991\n",
      "Epoche=849\n",
      "Action_times=39076\n",
      "Episolon=0.679777\n",
      "Epoche=850\n",
      "Action_times=39131\n",
      "Episolon=0.679409\n",
      "Epoche=851\n",
      "Action_times=39180\n",
      "Episolon=0.679081\n",
      "Epoche=852\n",
      "Action_times=39212\n",
      "Episolon=0.678867\n",
      "Epoche=853\n",
      "Action_times=39230\n",
      "Episolon=0.678746\n",
      "Epoche=854\n",
      "Action_times=39256\n",
      "Episolon=0.678573\n",
      "Epoche=855\n",
      "Action_times=39271\n",
      "Episolon=0.678472\n",
      "Epoche=856\n",
      "Action_times=39289\n",
      "Episolon=0.678352\n",
      "Epoche=857\n",
      "Action_times=39321\n",
      "Episolon=0.678138\n",
      "Epoche=858\n",
      "Action_times=39357\n",
      "Episolon=0.677898\n",
      "Epoche=859\n",
      "Action_times=39361\n",
      "Episolon=0.677871\n",
      "Epoche=860\n",
      "Action_times=39382\n",
      "Episolon=0.677731\n",
      "Epoche=861\n",
      "Action_times=39394\n",
      "Episolon=0.677651\n",
      "Epoche=862\n",
      "Action_times=39400\n",
      "Episolon=0.677611\n",
      "Epoche=863\n",
      "Action_times=39435\n",
      "Episolon=0.677377\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoche=864\n",
      "Action_times=39487\n",
      "Episolon=0.677030\n",
      "Epoche=865\n",
      "Action_times=39508\n",
      "Episolon=0.676890\n",
      "Epoche=866\n",
      "Action_times=39531\n",
      "Episolon=0.676737\n",
      "Epoche=867\n",
      "Action_times=39554\n",
      "Episolon=0.676583\n",
      "Epoche=868\n",
      "Action_times=39599\n",
      "Episolon=0.676283\n",
      "Epoche=869\n",
      "Action_times=39635\n",
      "Episolon=0.676043\n",
      "Epoche=870\n",
      "Action_times=39725\n",
      "Episolon=0.675444\n",
      "Epoche=871\n",
      "Action_times=39781\n",
      "Episolon=0.675072\n",
      "Epoche=872\n",
      "Action_times=39833\n",
      "Episolon=0.674726\n",
      "Epoche=873\n",
      "Action_times=39874\n",
      "Episolon=0.674454\n",
      "Epoche=874\n",
      "Action_times=39904\n",
      "Episolon=0.674254\n",
      "Epoche=875\n",
      "Action_times=39924\n",
      "Episolon=0.674121\n",
      "Epoche=876\n",
      "Action_times=39954\n",
      "Episolon=0.673922\n",
      "Epoche=877\n",
      "Action_times=40006\n",
      "Episolon=0.673577\n",
      "Epoche=878\n",
      "Action_times=40066\n",
      "Episolon=0.673179\n",
      "Epoche=879\n",
      "Action_times=40100\n",
      "Episolon=0.672954\n",
      "Epoche=880\n",
      "Action_times=40129\n",
      "Episolon=0.672761\n",
      "Epoche=881\n",
      "Action_times=40185\n",
      "Episolon=0.672390\n",
      "Epoche=882\n",
      "Action_times=40210\n",
      "Episolon=0.672225\n",
      "Epoche=883\n",
      "Action_times=40272\n",
      "Episolon=0.671814\n",
      "Epoche=884\n",
      "Action_times=40384\n",
      "Episolon=0.671073\n",
      "Epoche=885\n",
      "Action_times=40394\n",
      "Episolon=0.671007\n",
      "Epoche=886\n",
      "Action_times=40401\n",
      "Episolon=0.670961\n",
      "Epoche=887\n",
      "Action_times=40429\n",
      "Episolon=0.670776\n",
      "Epoche=888\n",
      "Action_times=40477\n",
      "Episolon=0.670459\n",
      "Epoche=889\n",
      "Action_times=40492\n",
      "Episolon=0.670360\n",
      "Epoche=890\n",
      "Action_times=40540\n",
      "Episolon=0.670043\n",
      "Epoche=891\n",
      "Action_times=40552\n",
      "Episolon=0.669964\n",
      "Epoche=892\n",
      "Action_times=40591\n",
      "Episolon=0.669706\n",
      "Epoche=893\n",
      "Action_times=40611\n",
      "Episolon=0.669575\n",
      "Epoche=894\n",
      "Action_times=40640\n",
      "Episolon=0.669383\n",
      "Epoche=895\n",
      "Action_times=40648\n",
      "Episolon=0.669331\n",
      "Epoche=896\n",
      "Action_times=40684\n",
      "Episolon=0.669093\n",
      "Epoche=897\n",
      "Action_times=40710\n",
      "Episolon=0.668922\n",
      "Epoche=898\n",
      "Action_times=40767\n",
      "Episolon=0.668546\n",
      "Epoche=899\n",
      "Action_times=40791\n",
      "Episolon=0.668388\n",
      "Epoche=900\n",
      "Action_times=40830\n",
      "Episolon=0.668132\n",
      "Epoche=901\n",
      "Action_times=40877\n",
      "Episolon=0.667822\n",
      "Epoche=902\n",
      "Action_times=40923\n",
      "Episolon=0.667520\n",
      "Epoche=903\n",
      "Action_times=40981\n",
      "Episolon=0.667139\n",
      "Epoche=904\n",
      "Action_times=41022\n",
      "Episolon=0.666869\n",
      "Epoche=905\n",
      "Action_times=41049\n",
      "Episolon=0.666692\n",
      "Epoche=906\n",
      "Action_times=41093\n",
      "Episolon=0.666403\n",
      "Epoche=907\n",
      "Action_times=41135\n",
      "Episolon=0.666127\n",
      "Epoche=908\n",
      "Action_times=41168\n",
      "Episolon=0.665911\n",
      "Epoche=909\n",
      "Action_times=41174\n",
      "Episolon=0.665872\n",
      "Epoche=910\n",
      "Action_times=41210\n",
      "Episolon=0.665635\n",
      "Epoche=911\n",
      "Action_times=41226\n",
      "Episolon=0.665531\n",
      "Epoche=912\n",
      "Action_times=41234\n",
      "Episolon=0.665478\n",
      "Epoche=913\n",
      "Action_times=41283\n",
      "Episolon=0.665157\n",
      "Epoche=914\n",
      "Action_times=41314\n",
      "Episolon=0.664954\n",
      "Epoche=915\n",
      "Action_times=41354\n",
      "Episolon=0.664692\n",
      "Epoche=916\n",
      "Action_times=41384\n",
      "Episolon=0.664496\n",
      "Epoche=917\n",
      "Action_times=41406\n",
      "Episolon=0.664352\n",
      "Epoche=918\n",
      "Action_times=41442\n",
      "Episolon=0.664116\n",
      "Epoche=919\n",
      "Action_times=41479\n",
      "Episolon=0.663874\n",
      "Epoche=920\n",
      "Action_times=41510\n",
      "Episolon=0.663672\n",
      "Epoche=921\n",
      "Action_times=41551\n",
      "Episolon=0.663404\n",
      "Epoche=922\n",
      "Action_times=41563\n",
      "Episolon=0.663325\n",
      "Epoche=923\n",
      "Action_times=41582\n",
      "Episolon=0.663201\n",
      "Epoche=924\n",
      "Action_times=41616\n",
      "Episolon=0.662979\n",
      "Epoche=925\n",
      "Action_times=41621\n",
      "Episolon=0.662946\n",
      "Epoche=926\n",
      "Action_times=41658\n",
      "Episolon=0.662705\n",
      "Epoche=927\n",
      "Action_times=41682\n",
      "Episolon=0.662548\n",
      "Epoche=928\n",
      "Action_times=41685\n",
      "Episolon=0.662529\n",
      "Epoche=929\n",
      "Action_times=41714\n",
      "Episolon=0.662339\n",
      "Epoche=930\n",
      "Action_times=41730\n",
      "Episolon=0.662235\n",
      "Epoche=931\n",
      "Action_times=41761\n",
      "Episolon=0.662033\n",
      "Epoche=932\n",
      "Action_times=41797\n",
      "Episolon=0.661798\n",
      "Epoche=933\n",
      "Action_times=41851\n",
      "Episolon=0.661446\n",
      "Epoche=934\n",
      "Action_times=41858\n",
      "Episolon=0.661401\n",
      "Epoche=935\n",
      "Action_times=41896\n",
      "Episolon=0.661153\n",
      "Epoche=936\n",
      "Action_times=41925\n",
      "Episolon=0.660964\n",
      "Epoche=937\n",
      "Action_times=41931\n",
      "Episolon=0.660925\n",
      "Epoche=938\n",
      "Action_times=41970\n",
      "Episolon=0.660672\n",
      "Epoche=939\n",
      "Action_times=41980\n",
      "Episolon=0.660606\n",
      "Epoche=940\n",
      "Action_times=42014\n",
      "Episolon=0.660385\n",
      "Epoche=941\n",
      "Action_times=42046\n",
      "Episolon=0.660177\n",
      "Epoche=942\n",
      "Action_times=42100\n",
      "Episolon=0.659826\n",
      "Epoche=943\n",
      "Action_times=42130\n",
      "Episolon=0.659631\n",
      "Epoche=944\n",
      "Action_times=42175\n",
      "Episolon=0.659339\n",
      "Epoche=945\n",
      "Action_times=42236\n",
      "Episolon=0.658943\n",
      "Epoche=946\n",
      "Action_times=42255\n",
      "Episolon=0.658820\n",
      "Epoche=947\n",
      "Action_times=42317\n",
      "Episolon=0.658418\n",
      "Epoche=948\n",
      "Action_times=42342\n",
      "Episolon=0.658256\n",
      "Epoche=949\n",
      "Action_times=42408\n",
      "Episolon=0.657828\n",
      "Epoche=950\n",
      "Action_times=42447\n",
      "Episolon=0.657575\n",
      "Epoche=951\n",
      "Action_times=42472\n",
      "Episolon=0.657413\n",
      "Epoche=952\n",
      "Action_times=42533\n",
      "Episolon=0.657019\n",
      "Epoche=953\n",
      "Action_times=42570\n",
      "Episolon=0.656779\n",
      "Epoche=954\n",
      "Action_times=42599\n",
      "Episolon=0.656592\n",
      "Epoche=955\n",
      "Action_times=42648\n",
      "Episolon=0.656275\n",
      "Epoche=956\n",
      "Action_times=42685\n",
      "Episolon=0.656036\n",
      "Epoche=957\n",
      "Action_times=42713\n",
      "Episolon=0.655855\n",
      "Epoche=958\n",
      "Action_times=42744\n",
      "Episolon=0.655655\n",
      "Epoche=959\n",
      "Action_times=42783\n",
      "Episolon=0.655403\n",
      "Epoche=960\n",
      "Action_times=42821\n",
      "Episolon=0.655158\n",
      "Epoche=961\n",
      "Action_times=42833\n",
      "Episolon=0.655080\n",
      "Epoche=962\n",
      "Action_times=42853\n",
      "Episolon=0.654951\n",
      "Epoche=963\n",
      "Action_times=42883\n",
      "Episolon=0.654758\n",
      "Epoche=964\n",
      "Action_times=42911\n",
      "Episolon=0.654577\n",
      "Epoche=965\n",
      "Action_times=42919\n",
      "Episolon=0.654526\n",
      "Epoche=966\n",
      "Action_times=42967\n",
      "Episolon=0.654217\n",
      "Epoche=967\n",
      "Action_times=43015\n",
      "Episolon=0.653907\n",
      "Epoche=968\n",
      "Action_times=43024\n",
      "Episolon=0.653849\n",
      "Epoche=969\n",
      "Action_times=43041\n",
      "Episolon=0.653740\n",
      "Epoche=970\n",
      "Action_times=43134\n",
      "Episolon=0.653142\n",
      "Epoche=971\n",
      "Action_times=43173\n",
      "Episolon=0.652891\n",
      "Epoche=972\n",
      "Action_times=43201\n",
      "Episolon=0.652711\n",
      "Epoche=973\n",
      "Action_times=43215\n",
      "Episolon=0.652621\n",
      "Epoche=974\n",
      "Action_times=43244\n",
      "Episolon=0.652435\n",
      "Epoche=975\n",
      "Action_times=43284\n",
      "Episolon=0.652178\n",
      "Epoche=976\n",
      "Action_times=43298\n",
      "Episolon=0.652088\n",
      "Epoche=977\n",
      "Action_times=43327\n",
      "Episolon=0.651902\n",
      "Epoche=978\n",
      "Action_times=43339\n",
      "Episolon=0.651825\n",
      "Epoche=979\n",
      "Action_times=43374\n",
      "Episolon=0.651600\n",
      "Epoche=980\n",
      "Action_times=43387\n",
      "Episolon=0.651517\n",
      "Epoche=981\n",
      "Action_times=43404\n",
      "Episolon=0.651407\n",
      "Epoche=982\n",
      "Action_times=43434\n",
      "Episolon=0.651215\n",
      "Epoche=983\n",
      "Action_times=43478\n",
      "Episolon=0.650933\n",
      "Epoche=984\n",
      "Action_times=43503\n",
      "Episolon=0.650773\n",
      "Epoche=985\n",
      "Action_times=43539\n",
      "Episolon=0.650542\n",
      "Epoche=986\n",
      "Action_times=43552\n",
      "Episolon=0.650459\n",
      "Epoche=987\n",
      "Action_times=43592\n",
      "Episolon=0.650203\n",
      "Epoche=988\n",
      "Action_times=43612\n",
      "Episolon=0.650075\n",
      "Epoche=989\n",
      "Action_times=43633\n",
      "Episolon=0.649940\n",
      "Epoche=990\n",
      "Action_times=43661\n",
      "Episolon=0.649761\n",
      "Epoche=991\n",
      "Action_times=43664\n",
      "Episolon=0.649742\n",
      "Epoche=992\n",
      "Action_times=43681\n",
      "Episolon=0.649633\n",
      "Epoche=993\n",
      "Action_times=43712\n",
      "Episolon=0.649435\n",
      "Epoche=994\n",
      "Action_times=43756\n",
      "Episolon=0.649154\n",
      "Epoche=995\n",
      "Action_times=43801\n",
      "Episolon=0.648866\n",
      "Epoche=996\n",
      "Action_times=43809\n",
      "Episolon=0.648815\n",
      "Epoche=997\n",
      "Action_times=43824\n",
      "Episolon=0.648719\n",
      "Epoche=998\n",
      "Action_times=43867\n",
      "Episolon=0.648445\n",
      "Epoche=999\n",
      "Action_times=43872\n",
      "Episolon=0.648413\n"
     ]
    }
   ],
   "source": [
    "%matplotlib qt5\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "pd.set_option(\"display.max_rows\",999)\n",
    "Alpha=0.1 #Learning Rate\n",
    "Beta=0.9 #The deceasing rate of the future gain\n",
    "#The rate of choosing a randon action ,Epsilon\n",
    "Epsilon_start=1\n",
    "Epsilon_stop=0.01\n",
    "Decay_rate=0.00001\n",
    "LengthOfMesh=10\n",
    "TotalNumberOfMesh=LengthOfMesh*LengthOfMesh #We assume that the mesh forme is 10*10\n",
    "Actions=['Left_three','Left_four','Left_six','Right_three','Right_four','Right_six','Up_three','Up_four','Up_six']\n",
    "Obstacle=[3,5,16,21,23,27,29,31,33,34,56,57,66,67,78,81,86]\n",
    "#Only the Six_Gait can cross this type of obstacles\n",
    "Obstacle_six=[6,7,22,28,44,55,68,82,83,92]\n",
    "#Only the Six_Gait and Four_Gait can cross this type of obstacles, howerer the Four_Gait is the best\n",
    "Obstacle_four=[8,10,11,12,15,24,30,40,45,52,72,79,85]\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Caculate_coordinate(Current_state):\n",
    "    x_axis=0.5+(Current_state-1)%LengthOfMesh\n",
    "    y_axis=0.5+((Current_state-1)//LengthOfMesh)\n",
    "    return x_axis,y_axis\n",
    "def Initial_Q_Table (NumberOfStates,Actions):\n",
    "    Q_Table=np.zeros([NumberOfStates,len(Actions)])\n",
    "    Q_Table=pd.DataFrame(Q_Table,index=np.arange(1,NumberOfStates+1),columns=Actions)\n",
    "    print ('We have finished initializing the Q-table') \n",
    "    return Q_Table\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "#According to the current state, we choose an action and then execute it.\n",
    "def Select_action(State,Q_Table,Action_times):\n",
    "    Epsilon=Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-Decay_rate*Action_times)\n",
    "    States_Actions=Q_Table.loc[[State],:]\n",
    "    if(np.random.rand()<Epsilon or np.all(States_Actions==[0])):\n",
    "        Execute_action=np.random.choice(Actions)#Select an action randomy\n",
    "    else:\n",
    "        Execute_action=np.array(pd.DataFrame.idxmax((Q_Table.T).loc[:,[State]]))[0]#Select the action having the maximun QValue\n",
    "    return Execute_action\n",
    "#———————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "#According to the current state and the selected current action, return the new state and rewards\n",
    "def Output_next_state(Current_state,Execute_action):\n",
    "    if(Execute_action=='Left_three'):\n",
    "        if(Current_state%LengthOfMesh==1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_four):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        else:\n",
    "            Next_state=Current_state-1\n",
    "            Reward=1\n",
    "    elif(Execute_action=='Left_four'):\n",
    "        if(Current_state%LengthOfMesh==1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_four):\n",
    "            Next_state=Current_state-1\n",
    "            Reward=1\n",
    "        else:\n",
    "            Next_state=Current_state-1\n",
    "            Reward=0.6\n",
    "    elif(Execute_action=='Left_six'):\n",
    "        if(Current_state%LengthOfMesh==1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_six):\n",
    "            Next_state=Current_state-1\n",
    "            Reward=1\n",
    "        elif((Current_state-1) in Obstacle_four):\n",
    "            Next_state=Current_state-1\n",
    "            Reward=0.6\n",
    "        else:\n",
    "            Next_state=Current_state-1\n",
    "            Reward=0.2\n",
    "    \n",
    "    elif(Execute_action=='Right_three'):\n",
    "        if(Current_state==TotalNumberOfMesh-1):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(Current_state%LengthOfMesh==0):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_four):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        else:\n",
    "            Next_state=Current_state+1\n",
    "            Reward=1\n",
    "    elif(Execute_action=='Right_four'):\n",
    "        if(Current_state==TotalNumberOfMesh-1):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(Current_state%LengthOfMesh==0):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_four):\n",
    "            Next_state=Current_state+1\n",
    "            Reward=1\n",
    "        else:\n",
    "            Next_state=Current_state+1\n",
    "            Reward=0.6\n",
    "    elif(Execute_action=='Right_six'):\n",
    "        if(Current_state==TotalNumberOfMesh-1):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(Current_state%LengthOfMesh==0):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_six):\n",
    "            Next_state=Current_state+1\n",
    "            Reward=1\n",
    "        elif((Current_state+1)in Obstacle_four):\n",
    "            Next_state=Current_state+1\n",
    "            Reward=0.6\n",
    "        else:\n",
    "            Next_state=Current_state+1\n",
    "            Reward=0.2\n",
    "    \n",
    "    elif(Execute_action=='Up_three'):\n",
    "        if(Current_state==TotalNumberOfMesh-LengthOfMesh):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(TotalNumberOfMesh+1-LengthOfMesh<=Current_state<=TotalNumberOfMesh-1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_four):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        else:\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=1\n",
    "    elif(Execute_action=='Up_four'):\n",
    "        if(Current_state==TotalNumberOfMesh-LengthOfMesh):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(TotalNumberOfMesh+1-LengthOfMesh<=Current_state<=TotalNumberOfMesh-1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_four):\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=1\n",
    "        else:\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=0.6\n",
    "    elif(Execute_action=='Up_six'):\n",
    "        if(Current_state==TotalNumberOfMesh-LengthOfMesh):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(TotalNumberOfMesh+1-LengthOfMesh<=Current_state<=TotalNumberOfMesh-1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_six):\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=1\n",
    "        elif((Current_state+10) in Obstacle_four):\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=0.6\n",
    "        else:\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=0.2\n",
    "    return Next_state,Reward\n",
    "        \n",
    "#————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Run_Function():\n",
    "    Q_Table=Initial_Q_Table(TotalNumberOfMesh,Actions)\n",
    "    Action_times=0 \n",
    "    Epoche=0\n",
    "    while Epoche<1000:\n",
    "        if(np.random.rand()<0.4):\n",
    "            Current_state=1\n",
    "        else:\n",
    "            Current_state=np.random.randint(low=5, high=100)\n",
    "        Is_end=False\n",
    "        while not Is_end:\n",
    "            Action=Select_action(Current_state,Q_Table,Action_times)\n",
    "            Next_state,Reward=Output_next_state(Current_state,Action)\n",
    "            Q_table_state_action=Q_Table.loc[[Current_state],[Action]]\n",
    "            if Next_state!='End':\n",
    "                Next_state_column=np.array((Q_Table.loc[[Next_state],:]).T)\n",
    "                Q_target=Reward+Beta*Next_state_column.max() #Next_S状态中的可选动作中的最大Q值动作\n",
    "                Action_times+=1\n",
    "            else:\n",
    "                print('Epoche=%d'%Epoche)\n",
    "                print('Action_times=%d'%Action_times)\n",
    "                print('Episolon=%f'%(Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-Decay_rate*Action_times)))\n",
    "                Q_target=Reward\n",
    "                Action_times+=1\n",
    "                Is_end=True \n",
    "            Q_Table.loc[[Current_state],[Action]]+=Alpha*(Q_target-Q_table_state_action)\n",
    "            Current_state=Next_state\n",
    "        Epoche+=1\n",
    " \n",
    "    return Q_Table\n",
    "\n",
    "Q__Table=Run_Function()\n",
    "Q__Table\n",
    "Final_Q_Table=np.array(Q__Table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def Plot_Final_Graph(Q_TABLE):\n",
    "    fig=plt.figure()\n",
    "    ax=fig.gca()\n",
    "    ax.set(xlim=[0, LengthOfMesh], ylim=[0, LengthOfMesh])\n",
    "    ax.set_xticks(np.arange(0,(LengthOfMesh+1)))\n",
    "    ax.set_yticks(np.arange(0,(LengthOfMesh+1)))\n",
    "    plt.grid()\n",
    "    Current_statestart=1\n",
    "    Current_state=Current_statestart\n",
    "    Action_times=1\n",
    "    while(Current_state!=TotalNumberOfMesh):\n",
    "        Action=np.argmax((Q_TABLE[Current_state-1]))\n",
    "        print('--------------------------------------------------')\n",
    "        print('Current_state=%d'%Current_state)\n",
    "        print('Action=%s'%(Actions[Action]))       \n",
    "        if(Action in [0,1,2]):\n",
    "             Next_state=Current_state-1\n",
    "        elif(Action in [3,4,5]):\n",
    "             Next_state=Current_state+1\n",
    "        else:\n",
    "             Next_state=Current_state+LengthOfMesh\n",
    "        print('Next_state=%d'%Next_state)\n",
    "        Action_times+=1\n",
    "        x_axis_current_state,y_axis_current_state=Caculate_coordinate(Current_state)\n",
    "        x_axis_next_state,y_axis_next_state=Caculate_coordinate(Next_state)\n",
    "        if(Action_times==0):\n",
    "            plt.scatter(x_axis_current_state,y_axis_current_state)  \n",
    "        else:\n",
    "            plt.scatter(x_axis_current_state,y_axis_current_state) \n",
    "            plt.scatter(x_axis_next_state,y_axis_next_state)\n",
    "            plt.plot([x_axis_current_state,x_axis_next_state],[y_axis_current_state,y_axis_next_state])\n",
    "        for J in np.arange(len(Obstacle)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle[J]))[0],(Caculate_coordinate(Obstacle[J]))[1],marker=\"x\",color='k',s=300)\n",
    "        for J in np.arange(len(Obstacle_six)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle_six[J]))[0],(Caculate_coordinate(Obstacle_six[J]))[1],marker=\"H\",color='k',s=300)\n",
    "        for J in np.arange(len(Obstacle_four)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle_four[J]))[0],(Caculate_coordinate(Obstacle_four[J]))[1],marker=\"s\",color='k',s=300)\n",
    "        Current_state=Next_state\n",
    "    plt.show()\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Plot_Final_Graph(Final_Q_Table)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
